diff --git a/add_defgroup_ingroup.py b/add_defgroup_ingroup.py new file mode 100644 index 00000000..50f4eca2 --- /dev/null +++ b/add_defgroup_ingroup.py @@ -0,0 +1,138 @@ +# #region AddDefgroupIngroup [C:3] [TYPE Module] [SEMANTICS migration,defgroup,ingroup] +# @defgroup Migration Add @defgroup/@ingroup — zero-risk additive operation. +# @BRIEF Inserts one line after each #region anchor. No renames. No deletions. Bottom-up insertion. +# @RATIONALE 97.9% of contracts lack @ingroup — the model's strongest pre-training DSA signal. +# @INVARIANT Anchor pairs untouched. @RELATION edges untouched. Business logic untouched. + +import re, sys +from pathlib import Path + +PATH_TO_DOMAIN = [ + ("frontend/src/lib/models/", "Models"), ("frontend/src/lib/components/ui/", "UI"), + ("frontend/src/lib/components/translate/", "Translate"), ("frontend/src/lib/components/dashboard/", "Dashboard"), + ("frontend/src/lib/components/llm/", "LLM"), ("frontend/src/lib/components/dataset-review/", "DatasetReview"), + ("frontend/src/lib/components/settings/", "Settings"), ("frontend/src/lib/components/layout/", "Layout"), + ("frontend/src/lib/components/assistant/", "Assistant"), ("frontend/src/lib/components/tasks/", "Tasks"), + ("frontend/src/lib/components/", "Components"), ("frontend/src/lib/ui/", "UI"), + ("frontend/src/lib/stores/", "Stores"), ("frontend/src/lib/api/", "ApiClient"), + ("frontend/src/routes/", "Routes"), ("frontend/src/types/", "Types"), + ("backend/src/api/routes/assistant/", "AssistantApi"), ("backend/src/api/routes/", "Api"), + ("backend/src/core/task_manager/", "TaskManager"), ("backend/src/core/auth/", "Auth"), + ("backend/src/core/migration/", "Migration"), ("backend/src/core/plugins/", "Plugin"), + ("backend/src/core/", "Core"), ("backend/src/services/dataset_review/", "DatasetReview"), + ("backend/src/services/", "Services"), ("backend/src/models/", "Models"), + ("backend/src/schemas/", "Schemas"), ("backend/src/plugins/translate/", "Translate"), + ("backend/src/plugins/llm_analysis/", "LLMAnalysis"), ("backend/src/plugins/git/", "Git"), + ("backend/src/plugins/storage/", "Storage"), ("backend/src/plugins/", "Plugin"), + ("backend/tests/", "Tests"), +] + +SEMANTICS_TO_DOMAIN = { + "auth": "Auth", "login": "Auth", "token": "Auth", "migration": "Migration", + "dashboard": "Dashboard", "dataset": "Dataset", "translate": "Translate", + "translation": "Translate", "llm": "LLM", "git": "Git", "storage": "Storage", + "plugin": "Plugin", "task": "Tasks", "test": "Tests", "api": "Api", "ui": "UI", + "users": "Users", "roles": "Roles", "settings": "Settings", "profile": "Profile", + "reports": "Reports", "validation": "Validation", "backup": "Backup", + "mapper": "Mapper", "debug": "Debug", "assistant": "Assistant", "admin": "Admin", + "notification": "Notifications", "schedule": "Schedule", "logging": "Logging", + "indexing": "Index", "curation": "Curator", "semantic": "Semantic", + "screenshot": "Screenshot", "health": "Health", +} + +ANCHOR_RE = re.compile( + r'^(\s*(?:#|//|" + elif stripped.startswith("//"): + prefix, suffix = "// ", "" + else: + prefix, suffix = "# ", "" + + is_mod = "[TYPE Module]" in anchor_line or "[TYPE Class]" in anchor_line + line = f"\n{prefix}@defgroup {domain} Module group.{suffix}" if is_mod else f"\n{prefix}@ingroup {domain}{suffix}" + insertions.append((pos, line)) + + # Apply bottom-up + new_content = content + for pos, line in sorted(insertions, reverse=True): + new_content = new_content[:pos] + line + new_content[pos:] + + if insertions and not dry_run: + fp.write_text(new_content, encoding="utf-8") + + return {"file": str(fp), "added": len(insertions)} + + +def main(): + dry = "--apply" not in sys.argv + root = Path.cwd() + for a in sys.argv[1:]: + if a.startswith("--root="): + root = Path(a.split("=", 1)[1]) + print(f"=== @defgroup/@ingroup — {'DRY RUN' if dry else 'APPLY'} ===") + files = find_files(root) + total, results = 0, [] + for f in files: + r = process(f, dry_run=dry) + if r["added"]: + results.append(r) + total += r["added"] + print(f"Files: {len(results)} Insertions: {total}") + if dry: + print("Run with --apply.") + +if __name__ == "__main__": + main() +# #endregion AddDefgroupIngroup diff --git a/backend/src/__init__.py b/backend/src/__init__.py index 94e38492..a1ddfcfa 100644 --- a/backend/src/__init__.py +++ b/backend/src/__init__.py @@ -1,3 +1,4 @@ # #region SrcRoot [TYPE Module] [SEMANTICS root, package] +# @defgroup Module Module group. # @BRIEF Canonical backend package root for application, scripts, and tests. # #endregion SrcRoot diff --git a/backend/src/api/__init__.py b/backend/src/api/__init__.py index 262e0118..765fbe32 100644 --- a/backend/src/api/__init__.py +++ b/backend/src/api/__init__.py @@ -1,3 +1,4 @@ # #region src.api [TYPE Package] [SEMANTICS api, package, init] +# @ingroup Module # @BRIEF Backend API package root. # #endregion src.api diff --git a/backend/src/api/auth.py b/backend/src/api/auth.py index 95738a4c..c8e8a816 100755 --- a/backend/src/api/auth.py +++ b/backend/src/api/auth.py @@ -1,14 +1,14 @@ -# #region AuthApi [C:5] [TYPE Module] [SEMANTICS fastapi, auth, api] -# @BRIEF Authentication API endpoints. +# #region Api.Auth [C:5] [TYPE Module] [SEMANTICS api,auth,fastapi] +# @defgroup Auth Authentication API endpoints — login, logout, token, ADFS. # @LAYER API -# @PRE Python environment and dependencies installed; database available. -# @POST FastAPI app instance with auth routes registered. +# @PRE Database available; JWT secret configured. +# @POST FastAPI routes registered for all auth operations. # @SIDE_EFFECT Registers API routes; configures OAuth and CORS middleware. -# @DATA_CONTRACT Input -> OAuth2PasswordRequestForm -> Token, User -# @INVARIANT All auth endpoints must return consistent error codes. -# @RELATION DEPENDS_ON -> [is_adfs_configured] -# @REVIEWED 2026-06-03 -# @TESTED true +# @DATA_CONTRACT OAuth2PasswordRequestForm -> Token | User +# @INVARIANT All auth endpoints return consistent error codes (401/403/422). +# @RELATION DEPENDS_ON -> [Auth.Jwt] +# @RELATION DEPENDS_ON -> [Auth.Service] +# @RELATION DEPENDS_ON -> [Auth.OAuth] from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordRequestForm @@ -32,12 +32,17 @@ router = APIRouter(prefix="/api/auth", tags=["auth"]) # #endregion router -# #region login_for_access_token [C:4] [TYPE Function] -# @BRIEF Authenticates a user and returns a JWT access token. +# #region Api.Auth.Login [C:4] [TYPE Function] [SEMANTICS api,auth,login,token] +# @ingroup Auth +# @BRIEF Authenticate user by credentials and return JWT access token. # @PRE form_data contains username and password. -# @POST Returns a Token object on success. -# @SIDE_EFFECT DB read/write for auth session; writes security event log. -# @RELATION CALLS -> [AuthService] +# @POST Returns Token(access_token, token_type) on success; 401 on failure. +# @SIDE_EFFECT DB read for user verification; writes security event LOGIN. +# @SIDE_EFFECT Molecular CoT: REASON on entry, REFLECT on success, EXPLORE on failure. +# @RELATION CALLS -> [Auth.Service] +# @TEST_EDGE: invalid_credentials -> 401 +# @TEST_EDGE: locked_account -> 423 +# @TEST_EDGE: missing_fields -> 422 @router.post("/login", response_model=Token) async def login_for_access_token( @@ -74,14 +79,15 @@ async def login_for_access_token( return auth_service.create_session(user) -# #endregion login_for_access_token +# #endregion Api.Auth.Login -# #region read_users_me [C:4] [TYPE Function] -# @BRIEF Retrieves the profile of the currently authenticated user. -# @PRE Valid JWT token provided. -# @POST Returns the current user's data. -# @RELATION DEPENDS_ON -> [get_current_user] -# @SIDE_EFFECT Reads current user from DB via auth middleware; writes security event log. +# #region Api.Auth.Me [C:3] [TYPE Function] [SEMANTICS api,auth,profile] +# @ingroup Auth +# @BRIEF Retrieve the profile of the currently authenticated user. +# @PRE Valid JWT token in Authorization header. +# @POST Returns UserSchema with id, username, email, roles. +# @SIDE_EFFECT Reads current user from DB via auth middleware. +# @RELATION DEPENDS_ON -> [Auth.Dependency.GetCurrentUser] @router.get("/me", response_model=UserSchema) async def read_users_me(current_user: UserSchema = Depends(get_current_user)): @@ -89,16 +95,19 @@ async def read_users_me(current_user: UserSchema = Depends(get_current_user)): return current_user -# #endregion read_users_me +# #endregion Api.Auth.Me -# #region logout [C:4] [TYPE Function] -# @BRIEF Logs out the current user — blacklists the JWT token server-side. -# @PRE Valid JWT token provided in Authorization header. -# @POST Token is added to blacklist; subsequent requests with same token are rejected. -# @SIDE_EFFECT Writes security event LOGOUT event; writes to token_blacklist table. -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION CALLS -> [blacklist_token] +# #region Api.Auth.Logout [C:4] [TYPE Function] [SEMANTICS api,auth,logout,revoke] +# @ingroup Auth +# @BRIEF Log out current user — blacklists the JWT token server-side. +# @PRE Valid JWT token in Authorization header. +# @POST Token added to blacklist; subsequent requests with same token rejected. +# @SIDE_EFFECT Writes security event LOGOUT; writes to token_blacklist table. +# @SIDE_EFFECT Molecular CoT: REASON/REFLECT/EXPLORE markers. +# @RELATION DEPENDS_ON -> [Auth.Dependency.GetCurrentUser] +# @RELATION CALLS -> [Auth.Jwt.BlacklistToken] +# @TEST_EDGE: already_expired_token -> 200 (idempotent) @router.post("/logout") async def logout( @@ -122,14 +131,15 @@ async def logout( return {"message": "Successfully logged out"} -# #endregion logout +# #endregion Api.Auth.Logout -# #region login_adfs [C:4] [TYPE Function] -# @BRIEF Initiates the ADFS OIDC login flow. -# @POST Redirects the user to ADFS. -# @RELATION CALLS -> [is_adfs_configured] -# @SIDE_EFFECT Redirects user to ADFS external OIDC provider. +# #region Api.Auth.LoginAdfs [C:3] [TYPE Function] [SEMANTICS api,auth,adfs,oidc] +# @ingroup Auth +# @BRIEF Initiate ADFS OIDC login flow — redirects user to identity provider. +# @POST Redirects user to ADFS authorization endpoint. +# @SIDE_EFFECT Redirects browser to external OIDC provider. +# @RELATION CALLS -> [Auth.OAuth] @router.get("/login/adfs") async def login_adfs(request: starlette.requests.Request): @@ -143,14 +153,18 @@ async def login_adfs(request: starlette.requests.Request): return await oauth.adfs.authorize_redirect(request, str(redirect_uri)) -# #endregion login_adfs +# #endregion Api.Auth.LoginAdfs -# #region auth_callback_adfs [C:4] [TYPE Function] -# @BRIEF Handles the callback from ADFS after successful authentication. -# @POST Provisions user JIT and returns session token. -# @SIDE_EFFECT Provisions user in DB, creates auth session, writes security event log. -# @RELATION CALLS -> [AuthService] +# #region Api.Auth.CallbackAdfs [C:4] [TYPE Function] [SEMANTICS api,auth,adfs,callback] +# @ingroup Auth +# @BRIEF Handle ADFS OIDC callback — provisions user JIT, returns session token. +# @POST Provisions user in DB (JIT), creates auth session. +# @SIDE_EFFECT DB write for user provisioning; writes security event LOGIN_ADFS. +# @SIDE_EFFECT Molecular CoT: REASON/REFLECT/EXPLORE markers. +# @RELATION CALLS -> [Auth.Service] +# @TEST_EDGE: adfs_timeout -> 504 +# @TEST_EDGE: invalid_state -> 401 @router.get("/callback/adfs", name="auth_callback_adfs") async def auth_callback_adfs( @@ -174,5 +188,5 @@ async def auth_callback_adfs( return auth_service.create_session(user) -# #endregion auth_callback_adfs -# #endregion AuthApi +# #endregion Api.Auth.CallbackAdfs +# #endregion Api.Auth diff --git a/backend/src/api/routes/__init__.py b/backend/src/api/routes/__init__.py index e410d362..4cc37b08 100755 --- a/backend/src/api/routes/__init__.py +++ b/backend/src/api/routes/__init__.py @@ -1,4 +1,5 @@ # #region ApiRoutesModule [C:5] [TYPE Module] [SEMANTICS api, package, router, lazy, import] +# @defgroup Api Module group. # @BRIEF Provide lazy route module loading to avoid heavyweight imports during tests. # @LAYER API # @RELATION CALLS -> [ApiRoutesGetAttr] @@ -8,6 +9,7 @@ # @INVARIANT Only names listed in __all__ are importable via __getattr__. # #region Route_Group_Contracts [C:3] [TYPE Block] +# @ingroup Api # @BRIEF Declare the canonical route-module registry used by lazy imports and app router inclusion. # @RELATION DEPENDS_ON -> [PluginsRouter] # @RELATION DEPENDS_ON -> [TasksRouter] @@ -45,6 +47,7 @@ __all__ = [ # #region ApiRoutesGetAttr [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Lazily import route module by attribute name. # @RELATION DEPENDS_ON -> [ApiRoutesModule] # @PRE name is module candidate exposed in __all__. diff --git a/backend/src/api/routes/__tests__/test_datasets.py b/backend/src/api/routes/__tests__/test_datasets_routes.py similarity index 100% rename from backend/src/api/routes/__tests__/test_datasets.py rename to backend/src/api/routes/__tests__/test_datasets_routes.py diff --git a/backend/src/api/routes/admin.py b/backend/src/api/routes/admin.py index 0702ea77..d6ffebb4 100644 --- a/backend/src/api/routes/admin.py +++ b/backend/src/api/routes/admin.py @@ -1,4 +1,5 @@ # #region AdminApi [C:5] [TYPE Module] [SEMANTICS fastapi, admin, api, rbac, user] +# @defgroup Api Module group. # # @BRIEF Admin API endpoints for user and role management. # @LAYER API @@ -35,6 +36,7 @@ from ...services.rbac_permission_catalog import ( ) # #region router [TYPE Variable] +# @ingroup Api # @RELATION DEPENDS_ON -> fastapi.APIRouter # @BRIEF APIRouter instance for admin routes. router = APIRouter(prefix="/api/admin", tags=["admin"]) @@ -42,6 +44,7 @@ router = APIRouter(prefix="/api/admin", tags=["admin"]) # #region list_users [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Lists all registered users. # @PRE Current user has 'Admin' role. # @POST Returns a list of UserSchema objects. @@ -59,6 +62,7 @@ async def list_users( # #region create_user [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Creates a new local user. # @PRE Current user has 'Admin' role. # @POST New user is created in the database. @@ -97,6 +101,7 @@ async def create_user( # #region update_user [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Updates an existing user. # @PRE Current user has 'Admin' role. # @POST User record is updated in the database. @@ -137,6 +142,7 @@ async def update_user( # #region delete_user [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Deletes a user. # @PRE Current user has 'Admin' role. # @POST User record is removed from the database. @@ -174,6 +180,7 @@ async def delete_user( # #region list_roles [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Lists all available roles. # @RELATION CALLS -> [Role] @router.get("/roles", response_model=list[RoleSchema]) @@ -188,6 +195,7 @@ async def list_roles( # #region create_role [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Creates a new system role with associated permissions. # @PRE Role name must be unique. # @POST New Role record is created in auth.db. @@ -225,6 +233,7 @@ async def create_role( # #region update_role [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Updates an existing role's metadata and permissions. # @PRE role_id must be a valid existing role UUID. # @POST Role record is updated in auth.db. @@ -268,6 +277,7 @@ async def update_role( # #region delete_role [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Removes a role from the system. # @PRE role_id must be a valid existing role UUID. # @POST Role record is removed from auth.db. @@ -294,6 +304,7 @@ async def delete_role( # #region list_permissions [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Lists all available system permissions for assignment. # @POST Returns a list of all PermissionSchema objects. # @RELATION CALLS -> backend.src.core.auth.repository.AuthRepository.list_permissions @@ -324,6 +335,7 @@ async def list_permissions( # #region list_ad_mappings [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Lists all AD Group to Role mappings. # @RELATION CALLS -> ADGroupMapping @router.get("/ad-mappings", response_model=list[ADGroupMappingSchema]) @@ -339,6 +351,7 @@ async def list_ad_mappings( # #region create_ad_mapping [C:2] [TYPE Function] +# @ingroup Api # @RELATION DEPENDS_ON -> [ADGroupMapping] # @RELATION DEPENDS_ON -> [get_auth_db] # @RELATION DEPENDS_ON -> [has_permission] diff --git a/backend/src/api/routes/admin_api_keys.py b/backend/src/api/routes/admin_api_keys.py index ba965c1d..9fbf20a9 100644 --- a/backend/src/api/routes/admin_api_keys.py +++ b/backend/src/api/routes/admin_api_keys.py @@ -1,4 +1,5 @@ # #region AdminApiKeyRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, admin, api_key, crud] +# @defgroup Api Module group. # @BRIEF Admin API endpoints for API key management — list, generate (one-time reveal), and revoke. # @LAYER API # @RELATION DEPENDS_ON -> [APIKeyModel] @@ -20,6 +21,7 @@ from ...dependencies import has_permission from ...models.api_key import APIKey # #region router [TYPE Variable] +# @ingroup Api # @BRIEF APIRouter for admin API key management routes. router = APIRouter(prefix="/api/admin/api-keys", tags=["admin", "api-keys"]) # #endregion router @@ -76,6 +78,7 @@ class ApiKeyRevokeResponse(BaseModel): # ── Routes ──────────────────────────────────────────────────── # #region list_api_keys [C:2] [TYPE Function] +# @ingroup Api # @BRIEF List all API keys — NEVER returns key_hash or raw_key. # @PRE Requires admin:settings WRITE permission. # @POST Returns list of ApiKeyListItem without sensitive fields. @@ -103,6 +106,7 @@ async def list_api_keys( # #region create_api_key [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Generate a new API key — returns raw key ONCE, never stored or retrievable again. # @PRE Requires admin:settings WRITE permission. name is required, at least one permission. # @POST Creates APIKey row with SHA-256 hash. Returns raw key in response. @@ -152,6 +156,7 @@ async def create_api_key( # #region revoke_api_key [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Revoke an API key by setting active=False. Preserves row for audit. # @PRE Requires admin:settings WRITE permission. # @POST Sets active=False on the key. Returns 404 if already revoked or not found. diff --git a/backend/src/api/routes/assistant/__init__.py b/backend/src/api/routes/assistant/__init__.py index b81dc830..903baf0a 100644 --- a/backend/src/api/routes/assistant/__init__.py +++ b/backend/src/api/routes/assistant/__init__.py @@ -1,4 +1,5 @@ # #region AssistantApi [C:5] [TYPE Module] [SEMANTICS assistant, api, package, llm, execution] +# @defgroup AssistantApi Module group. # @BRIEF API routes for LLM assistant command parsing and safe execution orchestration. # @LAYER API # @RELATION DEPENDS_ON -> [TaskManager] diff --git a/backend/src/api/routes/assistant/_admin_routes.py b/backend/src/api/routes/assistant/_admin_routes.py index 6db7dcbb..13b49f6b 100644 --- a/backend/src/api/routes/assistant/_admin_routes.py +++ b/backend/src/api/routes/assistant/_admin_routes.py @@ -1,4 +1,5 @@ # #region AssistantAdminRoutes [C:5] [TYPE Module] [SEMANTICS assistant, admin, route, audit, conversation] +# @defgroup AssistantApi Module group. # @BRIEF FastAPI route handlers for assistant admin operations — conversation listing, deletion, history, audit. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantRoutes] @@ -38,6 +39,7 @@ from ._schemas import ( # #region list_conversations [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Return paginated conversation list for current user with archived flag and last message preview. # @PRE Authenticated user context and valid pagination params. # @POST Conversations are grouped by conversation_id sorted by latest activity descending. @@ -134,6 +136,7 @@ async def list_conversations( # #region delete_conversation [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Soft-delete or hard-delete a conversation and clear its in-memory trace. # @PRE conversation_id belongs to current_user. # @POST Conversation records are removed from DB and CONVERSATIONS cache. @@ -180,6 +183,7 @@ async def delete_conversation( @router.get("/history") # #region get_history [TYPE Function] +# @ingroup AssistantApi # @BRIEF Retrieve paginated assistant conversation history for current user. # @PRE Authenticated user is available and page params are valid. # @POST Returns persistent messages and mirrored in-memory snapshot for diagnostics. @@ -251,6 +255,7 @@ async def get_history( @router.get("/audit") # #region get_assistant_audit [TYPE Function] +# @ingroup AssistantApi # @BRIEF Return assistant audit decisions for current user from persistent and in-memory stores. # @PRE User has tasks:READ permission. # @POST Audit payload is returned in reverse chronological order from DB. diff --git a/backend/src/api/routes/assistant/_command_parser.py b/backend/src/api/routes/assistant/_command_parser.py index 23a9be47..33975d71 100644 --- a/backend/src/api/routes/assistant/_command_parser.py +++ b/backend/src/api/routes/assistant/_command_parser.py @@ -1,4 +1,5 @@ # #region AssistantCommandParser [C:4] [TYPE Module] [SEMANTICS assistant, command, parser, nlu, intent] +# @defgroup AssistantApi Module group. # @BRIEF Deterministic RU/EN command text parser that converts user messages into intent payloads. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantResolvers] diff --git a/backend/src/api/routes/assistant/_dataset_review.py b/backend/src/api/routes/assistant/_dataset_review.py index d3224c75..ca1c58b2 100644 --- a/backend/src/api/routes/assistant/_dataset_review.py +++ b/backend/src/api/routes/assistant/_dataset_review.py @@ -1,4 +1,5 @@ # #region AssistantDatasetReview [C:4] [TYPE Module] [SEMANTICS assistant, dataset, review, context, intent] +# @defgroup AssistantApi Module group. # @BRIEF Dataset review context loading and intent planning for the assistant API. # @LAYER API # @RELATION DEPENDS_ON -> [DatasetReviewOrchestrator] diff --git a/backend/src/api/routes/assistant/_dataset_review_dispatch.py b/backend/src/api/routes/assistant/_dataset_review_dispatch.py index 0c5120e6..17b6ddcd 100644 --- a/backend/src/api/routes/assistant/_dataset_review_dispatch.py +++ b/backend/src/api/routes/assistant/_dataset_review_dispatch.py @@ -1,4 +1,5 @@ # #region AssistantDatasetReviewDispatch [C:4] [TYPE Module] [SEMANTICS assistant, dataset, review, dispatch, confirm] +# @defgroup AssistantApi Module group. # @BRIEF Dispatch and confirmation handling for dataset-review assistant intents. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantDatasetReview] diff --git a/backend/src/api/routes/assistant/_dispatch.py b/backend/src/api/routes/assistant/_dispatch.py index a616c810..c8002108 100644 --- a/backend/src/api/routes/assistant/_dispatch.py +++ b/backend/src/api/routes/assistant/_dispatch.py @@ -1,4 +1,5 @@ # #region AssistantDispatch [C:4] [TYPE Module] [SEMANTICS assistant, dispatch, confirm, execution, orchestration] +# @defgroup AssistantApi Module group. # @BRIEF Intent dispatch engine and backward-compat wrapper around the central tool registry. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] diff --git a/backend/src/api/routes/assistant/_history.py b/backend/src/api/routes/assistant/_history.py index aaeb740e..5e3bb2c6 100644 --- a/backend/src/api/routes/assistant/_history.py +++ b/backend/src/api/routes/assistant/_history.py @@ -1,4 +1,5 @@ # #region AssistantHistory [C:2] [TYPE Module] [SEMANTICS assistant, history, audit, persistence, conversation] +# @defgroup AssistantApi Module group. # @BRIEF Conversation history, audit trail, and confirmation persistence helpers for the assistant API. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantSchemas] diff --git a/backend/src/api/routes/assistant/_llm_planner.py b/backend/src/api/routes/assistant/_llm_planner.py index 01780ab4..cc4f27ef 100644 --- a/backend/src/api/routes/assistant/_llm_planner.py +++ b/backend/src/api/routes/assistant/_llm_planner.py @@ -1,4 +1,5 @@ # #region AssistantLlmPlanner [C:5] [TYPE Module] [SEMANTICS assistant, llm, planner, tool, catalog] +# @defgroup AssistantApi Module group. # @BRIEF LLM-based intent planning, tool catalog construction, and authorization for the assistant API. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantSchemas] diff --git a/backend/src/api/routes/assistant/_llm_planner_intent.py b/backend/src/api/routes/assistant/_llm_planner_intent.py index fef11760..63d911ea 100644 --- a/backend/src/api/routes/assistant/_llm_planner_intent.py +++ b/backend/src/api/routes/assistant/_llm_planner_intent.py @@ -1,4 +1,5 @@ # #region AssistantLlmPlannerIntent [C:5] [TYPE Module] [SEMANTICS assistant, llm, intent, planning, authorization] +# @defgroup AssistantApi Module group. # @BRIEF LLM-based intent planning and authorization for the assistant API — separated from tool catalog. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantLlmPlanner] diff --git a/backend/src/api/routes/assistant/_resolvers.py b/backend/src/api/routes/assistant/_resolvers.py index 5a54d575..5efcb5e4 100644 --- a/backend/src/api/routes/assistant/_resolvers.py +++ b/backend/src/api/routes/assistant/_resolvers.py @@ -1,4 +1,5 @@ # #region AssistantResolvers [C:2] [TYPE Module] [SEMANTICS assistant, resolver, lookup, environment, mapper] +# @defgroup AssistantApi Module group. # @BRIEF Environment, dashboard, provider, and task resolution utilities for the assistant API. # @LAYER API # @RELATION DEPENDS_ON -> [ConfigManager] diff --git a/backend/src/api/routes/assistant/_routes.py b/backend/src/api/routes/assistant/_routes.py index e532105b..ccee4343 100644 --- a/backend/src/api/routes/assistant/_routes.py +++ b/backend/src/api/routes/assistant/_routes.py @@ -1,4 +1,5 @@ # #region AssistantRoutes [C:5] [TYPE Module] [SEMANTICS assistant, api, route, chat, execution] +# @defgroup AssistantApi Module group. # @BRIEF FastAPI route handlers for the assistant API — message sending, confirmation, conversation management. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantSchemas] @@ -65,6 +66,7 @@ router = APIRouter(tags=["Assistant"]) @router.post("/messages", response_model=AssistantMessageResponse) # #region send_message [C:5] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Parse assistant command, enforce safety gates, and dispatch executable intent. # @DATA_CONTRACT Input[AssistantMessageRequest,User,TaskManager,ConfigManager,Session] -> Output[AssistantMessageResponse] # @RELATION DEPENDS_ON -> [_plan_intent_with_llm] @@ -163,6 +165,7 @@ async def send_message(request: AssistantMessageRequest, current_user: User=Depe "/confirmations/{confirmation_id}/confirm", response_model=AssistantMessageResponse ) # #region confirm_operation [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Execute previously requested risky operation after explicit user confirmation. # @PRE confirmation_id exists, belongs to current user, is pending, and not expired. # @POST Confirmation state becomes consumed and operation result is persisted in history. @@ -251,6 +254,7 @@ async def confirm_operation( "/confirmations/{confirmation_id}/cancel", response_model=AssistantMessageResponse ) # #region cancel_operation [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Cancel pending risky operation and mark confirmation token as cancelled. # @PRE confirmation_id exists, belongs to current user, and is still pending. # @POST Confirmation becomes cancelled and cannot be executed anymore. diff --git a/backend/src/api/routes/assistant/_schemas.py b/backend/src/api/routes/assistant/_schemas.py index e3ebb91f..2668f911 100644 --- a/backend/src/api/routes/assistant/_schemas.py +++ b/backend/src/api/routes/assistant/_schemas.py @@ -1,4 +1,5 @@ # #region AssistantSchemas [C:2] [TYPE Module] [SEMANTICS assistant, pydantic, schema, store, permission] +# @defgroup AssistantApi Module group. # @BRIEF Pydantic models, in-memory stores, and permission mappings for the assistant API. # @LAYER API # @RELATION CALLED_BY -> [AssistantHistory] diff --git a/backend/src/api/routes/assistant/_tool_backup.py b/backend/src/api/routes/assistant/_tool_backup.py index 24b73262..42255030 100644 --- a/backend/src/api/routes/assistant/_tool_backup.py +++ b/backend/src/api/routes/assistant/_tool_backup.py @@ -1,4 +1,5 @@ # #region AssistantToolBackup [C:3] [TYPE Module] [SEMANTICS assistant, tool, backup] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "run_backup" tool — run backup for environment or specific dashboard. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -26,6 +27,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_run_backup [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="run_backup", domain="backup", diff --git a/backend/src/api/routes/assistant/_tool_capabilities.py b/backend/src/api/routes/assistant/_tool_capabilities.py index ad9f1abe..e2dc7894 100644 --- a/backend/src/api/routes/assistant/_tool_capabilities.py +++ b/backend/src/api/routes/assistant/_tool_capabilities.py @@ -1,4 +1,5 @@ # #region AssistantToolCapabilities [C:3] [TYPE Module] [SEMANTICS assistant, tool, capabilities, catalog] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "show_capabilities" tool — lists available assistant commands and examples. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -31,6 +32,7 @@ _HUMAN_LABELS: dict[str, str] = { # #region handle_show_capabilities [C:2] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="show_capabilities", domain="assistant", diff --git a/backend/src/api/routes/assistant/_tool_commit.py b/backend/src/api/routes/assistant/_tool_commit.py index 00fab6bf..45fe860e 100644 --- a/backend/src/api/routes/assistant/_tool_commit.py +++ b/backend/src/api/routes/assistant/_tool_commit.py @@ -1,4 +1,5 @@ # #region AssistantToolCommit [C:3] [TYPE Module] [SEMANTICS assistant, tool, git, commit] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "commit_changes" tool — commit dashboard repository changes. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -24,6 +25,7 @@ from ._dispatch import _get_git_service # #region handle_commit_changes [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="commit_changes", domain="git", diff --git a/backend/src/api/routes/assistant/_tool_create_branch.py b/backend/src/api/routes/assistant/_tool_create_branch.py index 55231ea2..bc2ef9e4 100644 --- a/backend/src/api/routes/assistant/_tool_create_branch.py +++ b/backend/src/api/routes/assistant/_tool_create_branch.py @@ -1,4 +1,5 @@ # #region AssistantToolCreateBranch [C:3] [TYPE Module] [SEMANTICS assistant, tool, git, branch] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "create_branch" tool — create git branch for a dashboard. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -24,6 +25,7 @@ from ._dispatch import _get_git_service # #region handle_create_branch [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="create_branch", domain="git", diff --git a/backend/src/api/routes/assistant/_tool_deploy.py b/backend/src/api/routes/assistant/_tool_deploy.py index ca1cc400..1a3d08c8 100644 --- a/backend/src/api/routes/assistant/_tool_deploy.py +++ b/backend/src/api/routes/assistant/_tool_deploy.py @@ -1,4 +1,5 @@ # #region AssistantToolDeploy [C:3] [TYPE Module] [SEMANTICS assistant, tool, git, deploy] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "deploy_dashboard" tool — deploy dashboard to target environment. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -22,6 +23,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_deploy_dashboard [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="deploy_dashboard", domain="git", diff --git a/backend/src/api/routes/assistant/_tool_health_summary.py b/backend/src/api/routes/assistant/_tool_health_summary.py index 351b9c06..52af9070 100644 --- a/backend/src/api/routes/assistant/_tool_health_summary.py +++ b/backend/src/api/routes/assistant/_tool_health_summary.py @@ -1,4 +1,5 @@ # #region AssistantToolHealthSummary [C:3] [TYPE Module] [SEMANTICS assistant, tool, health, summary] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "get_health_summary" tool — get summary of dashboard health. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -22,6 +23,7 @@ from ._tool_registry import assistant_tool # #region handle_get_health_summary [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="get_health_summary", domain="health", diff --git a/backend/src/api/routes/assistant/_tool_list_environments.py b/backend/src/api/routes/assistant/_tool_list_environments.py index a8d6c76a..bce85939 100644 --- a/backend/src/api/routes/assistant/_tool_list_environments.py +++ b/backend/src/api/routes/assistant/_tool_list_environments.py @@ -1,4 +1,5 @@ # #region AssistantToolListEnvironments [C:2] [TYPE Module] [SEMANTICS assistant, tool, environments, list] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "list_environments" tool — show available Superset environments. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -19,6 +20,7 @@ from ._tool_registry import assistant_tool # #region handle_list_environments [C:2] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="list_environments", domain="environments", diff --git a/backend/src/api/routes/assistant/_tool_llm.py b/backend/src/api/routes/assistant/_tool_llm.py index 36148e53..69de9219 100644 --- a/backend/src/api/routes/assistant/_tool_llm.py +++ b/backend/src/api/routes/assistant/_tool_llm.py @@ -1,4 +1,5 @@ # #region AssistantToolLlm [C:2] [TYPE Module] [SEMANTICS assistant, tool, llm, providers, status] +# @defgroup AssistantApi Module group. # @BRIEF Handlers for LLM operations — list providers and check LLM status. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -21,6 +22,7 @@ from ._tool_registry import assistant_tool # #region handle_list_llm_providers [C:2] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="list_llm_providers", domain="llm", @@ -69,6 +71,7 @@ async def handle_list_llm_providers( # #region handle_get_llm_status [C:2] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="get_llm_status", domain="llm", diff --git a/backend/src/api/routes/assistant/_tool_llm_documentation.py b/backend/src/api/routes/assistant/_tool_llm_documentation.py index b1a0b989..3a75e850 100644 --- a/backend/src/api/routes/assistant/_tool_llm_documentation.py +++ b/backend/src/api/routes/assistant/_tool_llm_documentation.py @@ -1,4 +1,5 @@ # #region AssistantToolLlmDocumentation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, documentation] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "run_llm_documentation" tool — generate dataset documentation via LLM. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -22,6 +23,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_run_llm_documentation [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="run_llm_documentation", domain="llm", diff --git a/backend/src/api/routes/assistant/_tool_llm_validation.py b/backend/src/api/routes/assistant/_tool_llm_validation.py index da8da51e..7b1faa0c 100644 --- a/backend/src/api/routes/assistant/_tool_llm_validation.py +++ b/backend/src/api/routes/assistant/_tool_llm_validation.py @@ -1,4 +1,5 @@ # #region AssistantToolLlmValidation [C:3] [TYPE Module] [SEMANTICS assistant, tool, llm, validation, task] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "run_llm_validation" tool — create a validation task (policy) and trigger a run. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -35,6 +36,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_run_llm_validation [C:4] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="run_llm_validation", domain="llm", diff --git a/backend/src/api/routes/assistant/_tool_maintenance.py b/backend/src/api/routes/assistant/_tool_maintenance.py index 250db5da..dc44f82b 100644 --- a/backend/src/api/routes/assistant/_tool_maintenance.py +++ b/backend/src/api/routes/assistant/_tool_maintenance.py @@ -1,4 +1,5 @@ # #region AssistantToolMaintenance [C:3] [TYPE Module] [SEMANTICS assistant, tool, maintenance, events] +# @defgroup AssistantApi Module group. # @BRIEF Handlers for maintenance operations — list events, start maintenance, end maintenance. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -28,6 +29,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_list_maintenance_events [C:2] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="list_maintenance_events", domain="maintenance", @@ -151,6 +153,7 @@ async def handle_list_maintenance_events( # #region handle_start_maintenance [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="start_maintenance", domain="maintenance", @@ -254,6 +257,7 @@ async def handle_start_maintenance( # #region handle_end_maintenance [C:2] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="end_maintenance", domain="maintenance", diff --git a/backend/src/api/routes/assistant/_tool_migration.py b/backend/src/api/routes/assistant/_tool_migration.py index 37492170..066c9902 100644 --- a/backend/src/api/routes/assistant/_tool_migration.py +++ b/backend/src/api/routes/assistant/_tool_migration.py @@ -1,4 +1,5 @@ # #region AssistantToolMigration [C:4] [TYPE Module] [SEMANTICS assistant, tool, migration, execute] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "execute_migration" tool — run dashboard migration between environments. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -27,6 +28,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_execute_migration [C:4] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="execute_migration", domain="migration", diff --git a/backend/src/api/routes/assistant/_tool_registry.py b/backend/src/api/routes/assistant/_tool_registry.py index 8681cc9f..5eaee690 100644 --- a/backend/src/api/routes/assistant/_tool_registry.py +++ b/backend/src/api/routes/assistant/_tool_registry.py @@ -1,4 +1,5 @@ # #region AssistantToolRegistry [C:4] [TYPE Module] [SEMANTICS assistant, registry, tool, dispatch, catalog] +# @defgroup AssistantApi Module group. # @BRIEF Central `@assistant_tool` decorator-based registry: auto-generates LLM catalog, dispatches operations, # and enforces permissions. Replaces hand-maintained `_build_tool_catalog()`, `_dispatch_intent()`, # `INTENT_PERMISSION_CHECKS`, and `_SAFE_OPS` to eliminate cross-file desynchronisation. @@ -69,6 +70,7 @@ _tools: dict[str, AssistantTool] = {} # #region assistant_tool [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Decorator that registers a function as an assistant tool handler. # @POST Tool is registered in `_tools` by operation name. Handler gets `__assistant_tool__` attr. # @SIDE_EFFECT Mutates the module-level `_tools` dict. @@ -196,6 +198,7 @@ def get_permission_checks() -> dict[str, list[tuple[str, str]]]: # #region get_catalog [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Build LLM tool catalog from registry, filtered by user permissions. # @POST Returns list of dicts suitable for LLM planner prompt. def get_catalog( @@ -232,6 +235,7 @@ def get_catalog( # #region dispatch [C:2] [TYPE Function] +# @ingroup AssistantApi # @BRIEF Look up operation in registry and call its handler. # @PRE operation is a known registered tool. # @POST Returns (text, task_id, actions) from the handler. diff --git a/backend/src/api/routes/assistant/_tool_search_dashboards.py b/backend/src/api/routes/assistant/_tool_search_dashboards.py index 56e518b2..b96aeed9 100644 --- a/backend/src/api/routes/assistant/_tool_search_dashboards.py +++ b/backend/src/api/routes/assistant/_tool_search_dashboards.py @@ -1,4 +1,5 @@ # #region AssistantToolSearchDashboards [C:3] [TYPE Module] [SEMANTICS assistant, tool, dashboards, search] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "search_dashboards" tool — search and list dashboards from an environment. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -22,6 +23,7 @@ from ._tool_registry import assistant_tool # #region handle_search_dashboards [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="search_dashboards", domain="dashboards", diff --git a/backend/src/api/routes/assistant/_tool_task_status.py b/backend/src/api/routes/assistant/_tool_task_status.py index ea8845b6..30f3a9be 100644 --- a/backend/src/api/routes/assistant/_tool_task_status.py +++ b/backend/src/api/routes/assistant/_tool_task_status.py @@ -1,4 +1,5 @@ # #region AssistantToolTaskStatus [C:3] [TYPE Module] [SEMANTICS assistant, tool, status, task] +# @defgroup AssistantApi Module group. # @BRIEF Handler for the "get_task_status" tool — retrieve task status by id or latest user task. # @LAYER API # @RELATION DEPENDS_ON -> [AssistantToolRegistry] @@ -22,6 +23,7 @@ from ._tool_registry import _check_any_permission, assistant_tool # #region handle_get_task_status [C:3] [TYPE Function] +# @ingroup AssistantApi @assistant_tool( operation="get_task_status", domain="status", diff --git a/backend/src/api/routes/clean_release.py b/backend/src/api/routes/clean_release.py index 6aa822e4..4fb40d87 100644 --- a/backend/src/api/routes/clean_release.py +++ b/backend/src/api/routes/clean_release.py @@ -1,4 +1,5 @@ # #region CleanReleaseApi [C:4] [TYPE Module] [SEMANTICS fastapi, clean-release, api, compliance, candidate, release] +# @defgroup Api Module group. # @BRIEF Expose clean release endpoints for candidate preparation and subsequent compliance flow. # @LAYER API # @RELATION DEPENDS_ON -> [get_clean_release_repository] @@ -47,6 +48,7 @@ router = APIRouter(prefix="/api/clean-release", tags=["Clean Release"]) # #region PrepareCandidateRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for candidate preparation endpoint. class PrepareCandidateRequest(BaseModel): candidate_id: str = Field(min_length=1) @@ -59,6 +61,7 @@ class PrepareCandidateRequest(BaseModel): # #region StartCheckRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for clean compliance check run startup. class StartCheckRequest(BaseModel): candidate_id: str = Field(min_length=1) @@ -71,6 +74,7 @@ class StartCheckRequest(BaseModel): # #region RegisterCandidateRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for candidate registration endpoint. class RegisterCandidateRequest(BaseModel): id: str = Field(min_length=1) @@ -83,6 +87,7 @@ class RegisterCandidateRequest(BaseModel): # #region ImportArtifactsRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for candidate artifact import endpoint. class ImportArtifactsRequest(BaseModel): artifacts: list[dict[str, Any]] = Field(default_factory=list) @@ -92,6 +97,7 @@ class ImportArtifactsRequest(BaseModel): # #region BuildManifestRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for manifest build endpoint. class BuildManifestRequest(BaseModel): created_by: str = Field(default="system") @@ -101,6 +107,7 @@ class BuildManifestRequest(BaseModel): # #region CreateComplianceRunRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for compliance run creation with optional manifest pinning. class CreateComplianceRunRequest(BaseModel): requested_by: str = Field(min_length=1) @@ -111,6 +118,7 @@ class CreateComplianceRunRequest(BaseModel): # #region register_candidate_v2_endpoint [TYPE Function] +# @ingroup Api # @BRIEF Register a clean-release candidate for headless lifecycle. # @PRE Candidate identifier is unique. # @POST Candidate is persisted in DRAFT status. @@ -152,6 +160,7 @@ async def register_candidate_v2_endpoint( # #region import_candidate_artifacts_v2_endpoint [TYPE Function] +# @ingroup Api # @BRIEF Import candidate artifacts in headless flow. # @PRE Candidate exists and artifacts array is non-empty. # @POST Artifacts are persisted and candidate advances to PREPARED if it was DRAFT. @@ -210,6 +219,7 @@ async def import_candidate_artifacts_v2_endpoint( # #region build_candidate_manifest_v2_endpoint [TYPE Function] +# @ingroup Api # @BRIEF Build immutable manifest snapshot for prepared candidate. # @PRE Candidate exists and has imported artifacts. # @POST Returns created ManifestDTO with incremented version. @@ -254,6 +264,7 @@ async def build_candidate_manifest_v2_endpoint( # #region get_candidate_overview_v2_endpoint [TYPE Function] +# @ingroup Api # @BRIEF Return expanded candidate overview DTO for headless lifecycle visibility. # @PRE Candidate exists. # @POST Returns CandidateOverviewDTO built from the same repository state used by headless US1 endpoints. @@ -370,6 +381,7 @@ async def get_candidate_overview_v2_endpoint( # #region prepare_candidate_endpoint [TYPE Function] +# @ingroup Api # @BRIEF Prepare candidate with policy evaluation and deterministic manifest generation. # @PRE Candidate and active policy exist in repository. # @POST Returns preparation result including manifest reference and violations. @@ -404,6 +416,7 @@ async def prepare_candidate_endpoint( # #region start_check [TYPE Function] +# @ingroup Api # @BRIEF Start and finalize a clean compliance check run and persist report artifacts. # @PRE Active policy and candidate exist. # @POST Returns accepted payload with check_run_id and started_at. @@ -540,6 +553,7 @@ async def start_check( # #region get_check_status [TYPE Function] +# @ingroup Api # @BRIEF Return terminal/intermediate status payload for a check run. # @PRE check_run_id references an existing run. # @POST Deterministic payload shape includes checks and violations arrays. @@ -592,6 +606,7 @@ async def get_check_status( # #region get_report [TYPE Function] +# @ingroup Api # @BRIEF Return persisted compliance report by report_id. # @PRE report_id references an existing report. # @POST Returns serialized report object. diff --git a/backend/src/api/routes/clean_release_v2.py b/backend/src/api/routes/clean_release_v2.py index 8af3ac9a..6b426d45 100644 --- a/backend/src/api/routes/clean_release_v2.py +++ b/backend/src/api/routes/clean_release_v2.py @@ -1,4 +1,5 @@ # #region CleanReleaseV2Api [C:4] [TYPE Module] [SEMANTICS fastapi, clean-release, candidate, lifecycle, api] +# @defgroup Api Module group. # @BRIEF Redesigned clean release API for headless candidate lifecycle. # @LAYER API # @RELATION DEPENDS_ON -> [CleanReleaseRepository] @@ -62,6 +63,7 @@ class RevokeRequest(dict): # #region register_candidate [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Register a new release candidate. # @PRE Payload contains required fields (id, version, source_snapshot_ref, created_by). # @POST Candidate is saved in repository. @@ -97,6 +99,7 @@ async def register_candidate( # #region import_artifacts [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Associate artifacts with a release candidate. # @PRE Candidate exists. # @POST Artifacts are processed (placeholder). @@ -130,6 +133,7 @@ async def import_artifacts( # #region build_manifest [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Generate distribution manifest for a candidate. # @PRE Candidate exists. # @POST Manifest is created and saved. @@ -177,6 +181,7 @@ async def build_manifest( # #region approve_candidate_endpoint [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Endpoint to record candidate approval. # @RELATION CALLS -> [approve_candidate] @router.post("/candidates/{candidate_id}/approve") @@ -205,6 +210,7 @@ async def approve_candidate_endpoint( # #region reject_candidate_endpoint [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Endpoint to record candidate rejection. # @RELATION CALLS -> [reject_candidate] @router.post("/candidates/{candidate_id}/reject") @@ -233,6 +239,7 @@ async def reject_candidate_endpoint( # #region publish_candidate_endpoint [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Endpoint to publish an approved candidate. # @RELATION CALLS -> [publish_candidate] @router.post("/candidates/{candidate_id}/publish") @@ -277,6 +284,7 @@ async def publish_candidate_endpoint( # #region revoke_publication_endpoint [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Endpoint to revoke a previous publication. # @RELATION CALLS -> [revoke_publication] @router.post("/publications/{publication_id}/revoke") diff --git a/backend/src/api/routes/dashboards/__init__.py b/backend/src/api/routes/dashboards/__init__.py index 27cc3247..54d1180a 100644 --- a/backend/src/api/routes/dashboards/__init__.py +++ b/backend/src/api/routes/dashboards/__init__.py @@ -1,4 +1,5 @@ # #region DashboardsApi [C:5] [TYPE Module] [SEMANTICS dashboard, api, package, search, git, task] +# @defgroup Api Module group. # # @BRIEF API endpoints for the Dashboard Hub - listing dashboards with Git and task status # @LAYER API diff --git a/backend/src/api/routes/dashboards/_action_routes.py b/backend/src/api/routes/dashboards/_action_routes.py index 7edec0ef..c15a1c80 100644 --- a/backend/src/api/routes/dashboards/_action_routes.py +++ b/backend/src/api/routes/dashboards/_action_routes.py @@ -1,4 +1,5 @@ # #region DashboardActionRoutes [C:2] [TYPE Module] [SEMANTICS fastapi, dashboard, api, backup] +# @defgroup Api Module group. # @BRIEF Dashboard action route handlers — migrate, backup. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:frontend:DashboardRouter]] @@ -22,6 +23,7 @@ from ._schemas import ( # #region migrate_dashboards [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Trigger bulk migration of dashboards from source to target environment # @PRE User has permission plugin:migration:execute # @PRE source_env_id and target_env_id are valid environment IDs @@ -100,6 +102,7 @@ async def migrate_dashboards( # #region backup_dashboards [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Trigger bulk backup of dashboards with optional cron schedule # @PRE User has permission plugin:backup:execute # @PRE env_id is a valid environment ID diff --git a/backend/src/api/routes/dashboards/_detail_routes.py b/backend/src/api/routes/dashboards/_detail_routes.py index 0be4d1d8..d0b90667 100644 --- a/backend/src/api/routes/dashboards/_detail_routes.py +++ b/backend/src/api/routes/dashboards/_detail_routes.py @@ -1,4 +1,5 @@ # #region DashboardDetailRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, dashboard, api, transform, search, task] +# @defgroup Api Module group. # @BRIEF Dashboard detail, db-mappings, task history, thumbnail route handlers. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:frontend:DashboardRouter]] @@ -41,6 +42,7 @@ from ._schemas import ( # #region get_database_mappings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get database mapping suggestions between source and target environments # @PRE User has permission plugin:migration:read # @PRE source_env_id and target_env_id are valid environment IDs @@ -107,6 +109,7 @@ async def get_database_mappings( # #region get_dashboard_detail [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Fetch detailed dashboard info with related charts and datasets # @PRE env_id must be valid and dashboard ref (slug or id) must exist # @POST Returns dashboard detail payload for overview page @@ -166,6 +169,7 @@ async def get_dashboard_detail( # #region get_dashboard_tasks_history [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Returns history of backup and LLM validation tasks for a dashboard. # @PRE dashboard ref (slug or id) is valid. # @POST Response contains sorted task history (newest first). @@ -268,6 +272,7 @@ async def get_dashboard_tasks_history( # #region get_dashboard_thumbnail [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Proxies Superset dashboard thumbnail with cache support. # @RELATION CALLS -> [AsyncAPIClient] # @PRE env_id must exist. diff --git a/backend/src/api/routes/dashboards/_helpers.py b/backend/src/api/routes/dashboards/_helpers.py index a20d8fe9..279e92eb 100644 --- a/backend/src/api/routes/dashboards/_helpers.py +++ b/backend/src/api/routes/dashboards/_helpers.py @@ -1,4 +1,5 @@ # #region DashboardHelpers [C:2] [TYPE Module] [SEMANTICS fastapi, dashboard, api, search, filter] +# @defgroup Api Module group. # @BRIEF Basic helper functions for dashboard route handlers — slug resolution, filter normalization. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [SupersetClient] diff --git a/backend/src/api/routes/dashboards/_listing_routes.py b/backend/src/api/routes/dashboards/_listing_routes.py index f5502847..b2c99ab5 100644 --- a/backend/src/api/routes/dashboards/_listing_routes.py +++ b/backend/src/api/routes/dashboards/_listing_routes.py @@ -1,4 +1,5 @@ # #region DashboardListingRoutes [C:4] [TYPE Module] [SEMANTICS fastapi, dashboard, api, search] +# @defgroup Api Module group. # @BRIEF Dashboard listing route handler for Dashboard Hub. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:frontend:DashboardRouter]] @@ -35,6 +36,7 @@ from ._schemas import DashboardsResponse, EffectiveProfileFilter # #region get_dashboards [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Fetch list of dashboards from a specific environment with Git status and last task status # @PRE env_id must be a valid environment ID # @PRE page must be >= 1 if provided diff --git a/backend/src/api/routes/dashboards/_projection.py b/backend/src/api/routes/dashboards/_projection.py index 5b6a49fe..8762f042 100644 --- a/backend/src/api/routes/dashboards/_projection.py +++ b/backend/src/api/routes/dashboards/_projection.py @@ -1,4 +1,5 @@ # #region DashboardProjection [C:2] [TYPE Module] [SEMANTICS dashboard, api, transform, profile, filter] +# @defgroup Api Module group. # @BRIEF Dashboard response projection and profile-filter helpers for Dashboard Hub routes. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [SupersetClient] diff --git a/backend/src/api/routes/dashboards/_schemas.py b/backend/src/api/routes/dashboards/_schemas.py index 37f6e44e..92ee635b 100644 --- a/backend/src/api/routes/dashboards/_schemas.py +++ b/backend/src/api/routes/dashboards/_schemas.py @@ -21,6 +21,7 @@ class GitStatus(BaseModel): # #region LastTask [C:2] [TYPE DataClass] +# @ingroup Api # @BRIEF DTO for the most recent background task associated with a dashboard. class LastTask(BaseModel): task_id: str | None = None @@ -154,6 +155,7 @@ class DashboardTaskHistoryResponse(BaseModel): # #region DatabaseMapping [C:2] [TYPE DataClass] +# @ingroup Api # @BRIEF DTO for cross-environment database ID mapping. class DatabaseMapping(BaseModel): source_db: str diff --git a/backend/src/api/routes/dataset_review.py b/backend/src/api/routes/dataset_review.py index 174ca93b..78d03492 100644 --- a/backend/src/api/routes/dataset_review.py +++ b/backend/src/api/routes/dataset_review.py @@ -1,4 +1,5 @@ # #region DatasetReviewApi [C:3] [TYPE Module] [SEMANTICS dataset, review, api, facade] +# @defgroup Api Module group. # @BRIEF Thin facade re-exporting router and public symbols from decomposed dataset review API sub-modules. # @LAYER API # @RATIONALE Original 2484-line monolith violated INV_7 (400-line module limit) by 6x. Decomposed into _dependencies (DTOs/guards/serializers) and _routes (handlers). diff --git a/backend/src/api/routes/dataset_review_pkg/_dependencies.py b/backend/src/api/routes/dataset_review_pkg/_dependencies.py index 536aeed9..c32d02fd 100644 --- a/backend/src/api/routes/dataset_review_pkg/_dependencies.py +++ b/backend/src/api/routes/dataset_review_pkg/_dependencies.py @@ -1,4 +1,5 @@ # #region DatasetReviewDependencies [C:2] [TYPE Module] [SEMANTICS dataset, review, dependency, di, serialize] +# @defgroup Api Module group. # @BRIEF Dependency injection, serialization helpers, and feature-flag guards for dataset review endpoints. # @LAYER API diff --git a/backend/src/api/routes/dataset_review_pkg/_routes.py b/backend/src/api/routes/dataset_review_pkg/_routes.py index fa8adfaa..28fbb8f8 100644 --- a/backend/src/api/routes/dataset_review_pkg/_routes.py +++ b/backend/src/api/routes/dataset_review_pkg/_routes.py @@ -1,4 +1,5 @@ # #region DatasetReviewRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, dataset, review, feedback, route] +# @defgroup Api Module group. # @BRIEF Persist thumbs up/down feedback for clarification question/answer content. @@ -92,6 +93,7 @@ router = APIRouter(prefix="/api/dataset-orchestration", tags=["Dataset Orchestra # #region list_sessions [C:3] [TYPE Function] +# @ingroup Api # @BRIEF List resumable dataset review sessions for the current user. @router.get( "/sessions", @@ -137,6 +139,7 @@ async def list_sessions( # #region start_session [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Start a new dataset review session from a Superset link or dataset selection. @router.post( "/sessions", @@ -191,6 +194,7 @@ async def start_session( # #region get_session_detail [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Return the full accessible dataset review session aggregate. @router.get( "/sessions/{session_id}", @@ -214,6 +218,7 @@ async def get_session_detail( # #region update_session [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Update resumable lifecycle status for an owned session. @router.patch( "/sessions/{session_id}", @@ -263,6 +268,7 @@ async def update_session( # #region delete_session [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Archive or hard-delete a session owned by the current user. @router.delete( "/sessions/{session_id}", @@ -314,6 +320,7 @@ async def delete_session( # #region export_documentation [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Export documentation output for the current session. @router.get( "/sessions/{session_id}/exports/documentation", @@ -352,6 +359,7 @@ async def export_documentation( # #region export_validation [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Export validation findings for the current session. @router.get( "/sessions/{session_id}/exports/validation", @@ -390,6 +398,7 @@ async def export_validation( # #region get_clarification_state [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Return the current clarification session summary and active question payload. @router.get( "/sessions/{session_id}/clarification", @@ -426,6 +435,7 @@ async def get_clarification_state( # #region resume_clarification [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Resume clarification mode on the highest-priority unresolved question. @router.post( "/sessions/{session_id}/clarification/resume", @@ -463,6 +473,7 @@ async def resume_clarification( # #region record_clarification_answer [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Persist one clarification answer before advancing the active pointer. @router.post( "/sessions/{session_id}/clarification/answers", @@ -513,6 +524,7 @@ async def record_clarification_answer( # #region update_field_semantic [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Apply one field-level semantic candidate decision or manual override. @router.patch( "/sessions/{session_id}/fields/{field_id}/semantic", @@ -553,6 +565,7 @@ async def update_field_semantic( # #region lock_field_semantic [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Lock one semantic field against later automatic overwrite. @router.post( "/sessions/{session_id}/fields/{field_id}/lock", @@ -593,6 +606,7 @@ async def lock_field_semantic( # #region unlock_field_semantic [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Unlock one semantic field so later automated candidate application may replace it. @router.post( "/sessions/{session_id}/fields/{field_id}/unlock", @@ -636,6 +650,7 @@ async def unlock_field_semantic( # #region approve_batch_semantic_fields [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Approve multiple semantic candidate decisions in one batch. @router.post( "/sessions/{session_id}/fields/semantic/approve-batch", @@ -686,6 +701,7 @@ async def approve_batch_semantic_fields( # #region list_execution_mappings [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Return the current mapping-review set for one accessible session. @router.get( "/sessions/{session_id}/mappings", @@ -712,6 +728,7 @@ async def list_execution_mappings( # #region update_execution_mapping [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Persist one owner-authorized execution-mapping effective value override. @router.patch( "/sessions/{session_id}/mappings/{mapping_id}", @@ -777,6 +794,7 @@ async def update_execution_mapping( # #region approve_execution_mapping [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Explicitly approve a warning-sensitive mapping transformation. @router.post( "/sessions/{session_id}/mappings/{mapping_id}/approve", @@ -825,6 +843,7 @@ async def approve_execution_mapping( # #region approve_batch_execution_mappings [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Approve multiple warning-sensitive execution mappings in one batch. @router.post( "/sessions/{session_id}/mappings/approve-batch", @@ -877,6 +896,7 @@ async def approve_batch_execution_mappings( # #region trigger_preview_generation [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Trigger Superset-side preview compilation for the current owned execution context. @router.post( "/sessions/{session_id}/preview", @@ -938,6 +958,7 @@ async def trigger_preview_generation( # #region launch_dataset [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Execute the current owned session launch handoff and return audited SQL Lab run context. @router.post( "/sessions/{session_id}/launch", @@ -996,6 +1017,7 @@ async def launch_dataset( # #region record_field_feedback [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Persist thumbs up/down feedback for AI-assisted semantic field content. @router.post( "/sessions/{session_id}/fields/{field_id}/feedback", @@ -1040,6 +1062,7 @@ async def record_field_feedback( # #region record_clarification_feedback [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Persist thumbs up/down feedback for clarification question/answer content. @router.post( "/sessions/{session_id}/clarification/questions/{question_id}/feedback", diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 7ede9f02..4d712608 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -1,4 +1,5 @@ # #region DatasetsApi [C:5] [TYPE Module] [SEMANTICS fastapi, dataset, api, search, mapping, mapped-fields] +# @defgroup Api Module group. # # @BRIEF API endpoints for the Dataset Hub - listing datasets with mapping progress # @LAYER API @@ -84,6 +85,7 @@ class MetricItem(BaseModel): # #endregion MetricItem # #region DatasetDetailResponse [C:2] [TYPE DataClass] +# @ingroup Api # @BRIEF Detailed DTO for a dataset including columns, linked dashboards, and metrics. # @RELATION DEPENDS_ON -> [MetricItem] class DatasetDetailResponse(BaseModel): @@ -116,6 +118,7 @@ class StatsCounts(BaseModel): # #endregion StatsCounts # #region DatasetsResponse [C:2] [TYPE DataClass] +# @ingroup Api # @BRIEF Paginated response DTO for dataset listings with stats. # @RELATION DEPENDS_ON -> [StatsCounts] class DatasetsResponse(BaseModel): @@ -146,6 +149,7 @@ class MetricDescriptionUpdate(BaseModel): # #endregion MetricDescriptionUpdate # #region get_dataset_ids [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Fetch list of all dataset IDs from a specific environment (without pagination) # @PRE env_id must be a valid environment ID # @POST Returns a list of all dataset IDs @@ -194,6 +198,7 @@ async def get_dataset_ids( # #endregion get_dataset_ids # #region get_datasets [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Fetch list of datasets from a specific environment with mapping progress and stats. # @PRE env_id must be a valid environment ID # @PRE page must be >= 1 if provided @@ -321,6 +326,7 @@ class MapColumnsRequest(BaseModel): # #endregion MapColumnsRequest # #region map_columns [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Trigger bulk column mapping for datasets # @PRE User has permission plugin:mapper:execute # @PRE env_id is a valid environment ID @@ -394,6 +400,7 @@ class GenerateDocsRequest(BaseModel): # #endregion GenerateDocsRequest # #region generate_docs [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Trigger bulk documentation generation for datasets # @PRE User has permission plugin:llm_analysis:execute # @PRE env_id is a valid environment ID @@ -447,6 +454,7 @@ async def generate_docs( # #endregion generate_docs # #region get_dataset_detail [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get detailed dataset information including columns and linked dashboards # @PRE env_id is a valid environment ID # @PRE dataset_id is a valid dataset ID @@ -502,6 +510,7 @@ def _strip_html_tags(text: str) -> str: # #endregion _strip_html_tags # #region update_column_description [C:4] [TYPE Endpoint] +# @ingroup Api # @BRIEF Save description for a single dataset column. Internal: GET full dataset from Superset, modify one column's description, PUT back. # @PRE dataset_id and column_id must exist in the target environment. # @POST Column description in Superset is updated. Response confirms success. @@ -591,6 +600,7 @@ async def update_column_description( # #endregion update_column_description # #region update_metric_description [C:4] [TYPE Endpoint] +# @ingroup Api # @BRIEF Save description for a single dataset metric. Mirror of update_column_description for metrics. # @PRE dataset_id and metric_id must exist in the target environment. # @POST Metric description in Superset is updated. diff --git a/backend/src/api/routes/environments.py b/backend/src/api/routes/environments.py index 5c46b46e..069eb975 100644 --- a/backend/src/api/routes/environments.py +++ b/backend/src/api/routes/environments.py @@ -1,4 +1,5 @@ # #region EnvironmentsApi [C:5] [TYPE Module] [SEMANTICS fastapi, environment, api] +# @defgroup Api Module group. # # @BRIEF API endpoints for listing environments and their databases. # @LAYER API @@ -30,12 +31,14 @@ def _normalize_superset_env_url(raw_url: str) -> str: # #endregion _normalize_superset_env_url # #region ScheduleSchema [TYPE DataClass] +# @ingroup Api class ScheduleSchema(BaseModel): enabled: bool = False cron_expression: str = Field(..., pattern=r'^(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|((((\d+,)*\d+|(\d+(\/|-)\d+)|\d+|\*) ?){4,6})$') # #endregion ScheduleSchema # #region EnvironmentResponse [TYPE DataClass] +# @ingroup Api class EnvironmentResponse(BaseModel): id: str name: str @@ -46,6 +49,7 @@ class EnvironmentResponse(BaseModel): # #endregion EnvironmentResponse # #region DatabaseResponse [TYPE DataClass] +# @ingroup Api class DatabaseResponse(BaseModel): uuid: str database_name: str @@ -53,6 +57,7 @@ class DatabaseResponse(BaseModel): # #endregion DatabaseResponse # #region get_environments [TYPE Function] [SEMANTICS list, environments, config] +# @ingroup Api # @BRIEF List all configured environments. # @LAYER API # @PRE config_manager is injected via Depends. @@ -90,6 +95,7 @@ async def get_environments( # #endregion get_environments # #region update_environment_schedule [TYPE Function] [SEMANTICS update, schedule, backup, environment] +# @ingroup Api # @BRIEF Update backup schedule for an environment. # @LAYER API # @PRE Environment id exists, schedule is valid ScheduleSchema. @@ -121,6 +127,7 @@ async def update_environment_schedule( # #endregion update_environment_schedule # #region get_environment_databases [TYPE Function] [SEMANTICS fetch, databases, superset, environment] +# @ingroup Api # @BRIEF Fetch the list of databases from a specific environment. # @LAYER API # @PRE Environment id exists. diff --git a/backend/src/api/routes/git/__init__.py b/backend/src/api/routes/git/__init__.py index ccb865ab..0943879c 100644 --- a/backend/src/api/routes/git/__init__.py +++ b/backend/src/api/routes/git/__init__.py @@ -1,4 +1,5 @@ # #region GitPackage [C:5] [TYPE Module] [SEMANTICS git, api, package, sync] +# @defgroup Api Module group. # @BRIEF Package root for decomposed git routes. Re-exports all public symbols from submodules. # @LAYER API # @RELATION CALLS -> [GitPackage] diff --git a/backend/src/api/routes/git/_config_routes.py b/backend/src/api/routes/git/_config_routes.py index 4b6a1230..de1d19d0 100644 --- a/backend/src/api/routes/git/_config_routes.py +++ b/backend/src/api/routes/git/_config_routes.py @@ -1,4 +1,5 @@ # #region GitConfigRoutes [C:2] [TYPE Module] [SEMANTICS fastapi, git, api, search, connection] +# @defgroup Api Module group. # @BRIEF FastAPI endpoints for Git server configuration CRUD and connection testing. # @LAYER API @@ -21,6 +22,7 @@ from ._router import router # #region get_git_configs [C:2] [TYPE Function] +# @ingroup Api # @BRIEF List all configured Git servers. @router.get("/config", response_model=list[GitServerConfigSchema]) async def get_git_configs( @@ -39,6 +41,7 @@ async def get_git_configs( # #region create_git_config [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Register a new Git server configuration. @router.post("/config", response_model=GitServerConfigSchema) async def create_git_config( @@ -57,6 +60,7 @@ async def create_git_config( # #region update_git_config [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Update an existing Git server configuration. @router.put("/config/{config_id}", response_model=GitServerConfigSchema) async def update_git_config( @@ -89,6 +93,7 @@ async def update_git_config( # #region delete_git_config [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Remove a Git server configuration. @router.delete("/config/{config_id}") async def delete_git_config( @@ -110,6 +115,7 @@ async def delete_git_config( # #region test_git_config [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Validate connection to a Git server using provided credentials. @router.post("/config/test") async def test_git_config( diff --git a/backend/src/api/routes/git/_environment_routes.py b/backend/src/api/routes/git/_environment_routes.py index 767ac9ce..18fe7b08 100644 --- a/backend/src/api/routes/git/_environment_routes.py +++ b/backend/src/api/routes/git/_environment_routes.py @@ -1,4 +1,5 @@ # #region GitEnvironmentRoutes [C:2] [TYPE Module] [SEMANTICS fastapi, git, environment, api] +# @defgroup Api Module group. # @BRIEF FastAPI endpoint for listing deployment environments. # @LAYER API @@ -13,6 +14,7 @@ from ._router import router # #region get_environments [C:2] [TYPE Function] +# @ingroup Api # @BRIEF List all deployment environments. @router.get("/environments", response_model=list[DeploymentEnvironmentSchema]) async def get_environments( diff --git a/backend/src/api/routes/git/_gitea_routes.py b/backend/src/api/routes/git/_gitea_routes.py index 172067b9..d8631dbf 100644 --- a/backend/src/api/routes/git/_gitea_routes.py +++ b/backend/src/api/routes/git/_gitea_routes.py @@ -1,4 +1,5 @@ # #region GitGiteaRoutes [C:2] [TYPE Module] [SEMANTICS fastapi, git, gitea, api] +# @defgroup Api Module group. # @BRIEF FastAPI endpoints for Gitea-specific repository operations. # @LAYER API @@ -23,6 +24,7 @@ from ._router import router # #region list_gitea_repositories [C:2] [TYPE Function] +# @ingroup Api # @BRIEF List repositories in Gitea for a saved Gitea config. @router.get("/config/{config_id}/gitea/repos", response_model=list[GiteaRepoSchema]) async def list_gitea_repositories( @@ -54,6 +56,7 @@ async def list_gitea_repositories( # #region create_gitea_repository [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Create a repository in Gitea for a saved Gitea config. @router.post("/config/{config_id}/gitea/repos", response_model=GiteaRepoSchema) async def create_gitea_repository( @@ -91,6 +94,7 @@ async def create_gitea_repository( # #region delete_gitea_repository [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Delete repository in Gitea for a saved Gitea config. @router.delete("/config/{config_id}/gitea/repos/{owner}/{repo_name}") async def delete_gitea_repository( @@ -118,6 +122,7 @@ async def delete_gitea_repository( # #region create_remote_repository [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Create repository on remote Git server using selected provider config. @router.post("/config/{config_id}/repositories", response_model=RemoteRepoSchema) async def create_remote_repository( diff --git a/backend/src/api/routes/git/_helpers.py b/backend/src/api/routes/git/_helpers.py index 652e608b..185adb0a 100644 --- a/backend/src/api/routes/git/_helpers.py +++ b/backend/src/api/routes/git/_helpers.py @@ -1,4 +1,5 @@ # #region GitHelpers [C:3] [TYPE Module] [SEMANTICS fastapi, git, api] +# @defgroup Api Module group. # @BRIEF Shared helper functions for Git route modules. # @LAYER API # @RELATION CALLS -> [GitDeps] diff --git a/backend/src/api/routes/git/_merge_routes.py b/backend/src/api/routes/git/_merge_routes.py index dd755750..45fd04da 100644 --- a/backend/src/api/routes/git/_merge_routes.py +++ b/backend/src/api/routes/git/_merge_routes.py @@ -1,4 +1,5 @@ # #region GitMergeRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, git, api] +# @defgroup Api Module group. # @BRIEF FastAPI endpoints for merge operations (status, conflicts, resolve, abort, continue). # @LAYER API @@ -22,6 +23,7 @@ from ._router import router # #region get_merge_status [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Return unfinished-merge status for repository (web-only recovery support). @router.get( "/repositories/{dashboard_ref}/merge/status", response_model=MergeStatusSchema @@ -49,6 +51,7 @@ async def get_merge_status( # #region get_merge_conflicts [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Return conflicted files with mine/theirs previews for web conflict resolver. @router.get( "/repositories/{dashboard_ref}/merge/conflicts", @@ -77,6 +80,7 @@ async def get_merge_conflicts( # #region resolve_merge_conflicts [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Apply mine/theirs/manual conflict resolutions from WebUI and stage files. # @RELATION CALLS -> [GitService] @router.post("/repositories/{dashboard_ref}/merge/resolve") @@ -108,6 +112,7 @@ async def resolve_merge_conflicts( # #region abort_merge [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Abort unfinished merge from WebUI flow. @router.post("/repositories/{dashboard_ref}/merge/abort") async def abort_merge( @@ -133,6 +138,7 @@ async def abort_merge( # #region continue_merge [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Finalize unfinished merge from WebUI flow. # @RELATION CALLS -> [GitService] @router.post("/repositories/{dashboard_ref}/merge/continue") diff --git a/backend/src/api/routes/git/_repo_lifecycle_routes.py b/backend/src/api/routes/git/_repo_lifecycle_routes.py index 8af9ec11..1c8b4d18 100644 --- a/backend/src/api/routes/git/_repo_lifecycle_routes.py +++ b/backend/src/api/routes/git/_repo_lifecycle_routes.py @@ -1,4 +1,5 @@ # #region GitRepoLifecycleRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, git, api, sync, deploy] +# @defgroup Api Module group. # @BRIEF FastAPI endpoints for Git lifecycle operations (sync, promote, deploy). # @LAYER API @@ -27,6 +28,7 @@ from ._router import router # #region sync_dashboard [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Sync dashboard state from Superset to Git using the GitPlugin. # @RELATION CALLS -> [GitPlugin] @router.post("/repositories/{dashboard_ref}/sync") @@ -62,6 +64,7 @@ async def sync_dashboard( # #region promote_dashboard [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Promote changes between branches via MR or direct merge. # @RELATION CALLS -> [GitPlugin] @router.post("/repositories/{dashboard_ref}/promote", response_model=PromoteResponse) @@ -185,6 +188,7 @@ async def promote_dashboard( # #region deploy_dashboard [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Deploy dashboard from Git to a target environment. # @RELATION CALLS -> [GitPlugin] @router.post("/repositories/{dashboard_ref}/deploy") diff --git a/backend/src/api/routes/git/_repo_operations_routes.py b/backend/src/api/routes/git/_repo_operations_routes.py index a57b4bb0..d8894d67 100644 --- a/backend/src/api/routes/git/_repo_operations_routes.py +++ b/backend/src/api/routes/git/_repo_operations_routes.py @@ -1,4 +1,5 @@ # #region GitRepoOperationsRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, git, api, history, diff] +# @defgroup Api Module group. # @BRIEF FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history). # @LAYER API @@ -29,6 +30,7 @@ from ._router import router # #region commit_changes [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Stage and commit changes in the dashboard's repository. @router.post("/repositories/{dashboard_ref}/commit") async def commit_changes( @@ -61,6 +63,7 @@ async def commit_changes( # #region push_changes [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Push local commits to the remote repository. @router.post("/repositories/{dashboard_ref}/push") async def push_changes( @@ -87,6 +90,7 @@ async def push_changes( # #region pull_changes [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Pull changes from the remote repository. # @RELATION CALLS -> [GitService] @router.post("/repositories/{dashboard_ref}/pull") @@ -149,6 +153,7 @@ async def pull_changes( # #region get_repository_status [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get current Git status for a dashboard repository. @router.get("/repositories/{dashboard_ref}/status") async def get_repository_status( @@ -173,6 +178,7 @@ async def get_repository_status( # #region get_repository_status_batch [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get Git statuses for multiple dashboard repositories in one request. @router.post("/repositories/status/batch", response_model=RepoStatusBatchResponse) async def get_repository_status_batch( @@ -212,6 +218,7 @@ async def get_repository_status_batch( # #region get_repository_diff [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get Git diff for a dashboard repository. @router.get("/repositories/{dashboard_ref}/diff") async def get_repository_diff( @@ -239,6 +246,7 @@ async def get_repository_diff( # #region get_history [C:2] [TYPE Function] +# @ingroup Api # @BRIEF View commit history for a dashboard's repository. @router.get("/repositories/{dashboard_ref}/history", response_model=list[CommitSchema]) async def get_history( @@ -265,6 +273,7 @@ async def get_history( # #region generate_commit_message [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Generate a suggested commit message using LLM. # @RELATION CALLS -> [GitService] @router.post("/repositories/{dashboard_ref}/generate-message") diff --git a/backend/src/api/routes/git/_repo_routes.py b/backend/src/api/routes/git/_repo_routes.py index 4f142913..d53bf60b 100644 --- a/backend/src/api/routes/git/_repo_routes.py +++ b/backend/src/api/routes/git/_repo_routes.py @@ -1,4 +1,5 @@ # #region GitRepoRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, git, api, search] +# @defgroup Api Module group. # @BRIEF FastAPI endpoints for core Git repository operations (init, binding, branches, checkout). # @LAYER API @@ -29,6 +30,7 @@ from ._router import router # #region init_repository [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Link a dashboard to a Git repository and perform initial clone/init. # @RELATION CALLS -> [GitService] @router.post("/repositories/{dashboard_ref}/init") @@ -109,6 +111,7 @@ async def init_repository( # #region get_repository_binding [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Return repository binding with provider metadata for selected dashboard. @router.get("/repositories/{dashboard_ref}", response_model=RepositoryBindingSchema) async def get_repository_binding( @@ -150,6 +153,7 @@ async def get_repository_binding( # #region delete_repository [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Delete local repository workspace and DB binding for selected dashboard. @router.delete("/repositories/{dashboard_ref}") async def delete_repository( @@ -176,6 +180,7 @@ async def delete_repository( # #region get_branches [C:2] [TYPE Function] +# @ingroup Api # @BRIEF List all branches for a dashboard's repository. @router.get("/repositories/{dashboard_ref}/branches", response_model=list[BranchSchema]) async def get_branches( @@ -201,6 +206,7 @@ async def get_branches( # #region create_branch [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Create a new branch in the dashboard's repository. @router.post("/repositories/{dashboard_ref}/branches") async def create_branch( @@ -233,6 +239,7 @@ async def create_branch( # #region checkout_branch [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Switch the dashboard's repository to a specific branch. @router.post("/repositories/{dashboard_ref}/checkout") async def checkout_branch( diff --git a/backend/src/api/routes/git_schemas.py b/backend/src/api/routes/git_schemas.py index 897208ad..a8dee3c9 100644 --- a/backend/src/api/routes/git_schemas.py +++ b/backend/src/api/routes/git_schemas.py @@ -27,6 +27,7 @@ class GitServerConfigBase(BaseModel): # #endregion GitServerConfigBase # #region GitServerConfigUpdate [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for updating an existing Git server configuration. class GitServerConfigUpdate(BaseModel): name: str | None = Field(None, description="Display name for the Git server") @@ -38,6 +39,7 @@ class GitServerConfigUpdate(BaseModel): # #endregion GitServerConfigUpdate # #region GitServerConfigCreate [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for creating a new Git server configuration. class GitServerConfigCreate(GitServerConfigBase): """Schema for creating a new Git server configuration.""" @@ -45,6 +47,7 @@ class GitServerConfigCreate(GitServerConfigBase): # #endregion GitServerConfigCreate # #region GitServerConfigSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for representing a Git server configuration with metadata. class GitServerConfigSchema(GitServerConfigBase): """Schema for representing a Git server configuration with metadata.""" @@ -56,6 +59,7 @@ class GitServerConfigSchema(GitServerConfigBase): # #endregion GitServerConfigSchema # #region GitRepositorySchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for tracking a local Git repository linked to a dashboard. class GitRepositorySchema(BaseModel): """Schema for tracking a local Git repository linked to a dashboard.""" @@ -71,6 +75,7 @@ class GitRepositorySchema(BaseModel): # #endregion GitRepositorySchema # #region BranchSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for representing a Git branch metadata. class BranchSchema(BaseModel): """Schema for representing a Git branch.""" @@ -81,6 +86,7 @@ class BranchSchema(BaseModel): # #endregion BranchSchema # #region CommitSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for representing Git commit details. class CommitSchema(BaseModel): """Schema for representing a Git commit.""" @@ -93,6 +99,7 @@ class CommitSchema(BaseModel): # #endregion CommitSchema # #region BranchCreate [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for branch creation requests. class BranchCreate(BaseModel): """Schema for branch creation requests.""" @@ -101,6 +108,7 @@ class BranchCreate(BaseModel): # #endregion BranchCreate # #region BranchCheckout [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for branch checkout requests. class BranchCheckout(BaseModel): """Schema for branch checkout requests.""" @@ -108,6 +116,7 @@ class BranchCheckout(BaseModel): # #endregion BranchCheckout # #region CommitCreate [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for staging and committing changes. class CommitCreate(BaseModel): """Schema for staging and committing changes.""" @@ -116,6 +125,7 @@ class CommitCreate(BaseModel): # #endregion CommitCreate # #region ConflictResolution [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for resolving merge conflicts. class ConflictResolution(BaseModel): """Schema for resolving merge conflicts.""" @@ -126,6 +136,7 @@ class ConflictResolution(BaseModel): # #region MergeStatusSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema representing unfinished merge status for repository. class MergeStatusSchema(BaseModel): has_unfinished_merge: bool @@ -139,6 +150,7 @@ class MergeStatusSchema(BaseModel): # #region MergeConflictFileSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema describing one conflicted file with optional side snapshots. class MergeConflictFileSchema(BaseModel): file_path: str @@ -148,6 +160,7 @@ class MergeConflictFileSchema(BaseModel): # #region MergeResolveRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for resolving one or multiple merge conflicts. class MergeResolveRequest(BaseModel): resolutions: list[ConflictResolution] = Field(default_factory=list) @@ -155,12 +168,14 @@ class MergeResolveRequest(BaseModel): # #region MergeContinueRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for finishing merge with optional explicit commit message. class MergeContinueRequest(BaseModel): message: str | None = None # #endregion MergeContinueRequest # #region DeploymentEnvironmentSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for representing a target deployment environment. class DeploymentEnvironmentSchema(BaseModel): """Schema for representing a target deployment environment.""" @@ -173,6 +188,7 @@ class DeploymentEnvironmentSchema(BaseModel): # #endregion DeploymentEnvironmentSchema # #region DeployRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for dashboard deployment requests. class DeployRequest(BaseModel): """Schema for deployment requests.""" @@ -180,6 +196,7 @@ class DeployRequest(BaseModel): # #endregion DeployRequest # #region RepoInitRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for repository initialization requests. class RepoInitRequest(BaseModel): """Schema for repository initialization requests.""" @@ -189,6 +206,7 @@ class RepoInitRequest(BaseModel): # #region RepositoryBindingSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema describing repository-to-config binding and provider metadata. class RepositoryBindingSchema(BaseModel): dashboard_id: int @@ -199,6 +217,7 @@ class RepositoryBindingSchema(BaseModel): # #endregion RepositoryBindingSchema # #region RepoStatusBatchRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for requesting repository statuses for multiple dashboards in a single call. class RepoStatusBatchRequest(BaseModel): dashboard_ids: list[int] = Field(default_factory=list, description="Dashboard IDs to resolve repository statuses for") @@ -206,6 +225,7 @@ class RepoStatusBatchRequest(BaseModel): # #region RepoStatusBatchResponse [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema for returning repository statuses keyed by dashboard ID. class RepoStatusBatchResponse(BaseModel): statuses: dict[str, dict[str, Any]] @@ -213,6 +233,7 @@ class RepoStatusBatchResponse(BaseModel): # #region GiteaRepoSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Schema describing a Gitea repository. class GiteaRepoSchema(BaseModel): name: str @@ -226,6 +247,7 @@ class GiteaRepoSchema(BaseModel): # #region GiteaRepoCreateRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for creating a Gitea repository. class GiteaRepoCreateRequest(BaseModel): name: str = Field(..., min_length=1, max_length=255) @@ -237,6 +259,7 @@ class GiteaRepoCreateRequest(BaseModel): # #region RemoteRepoSchema [TYPE Class] +# @defgroup Api Module group. # @BRIEF Provider-agnostic remote repository payload. class RemoteRepoSchema(BaseModel): provider: GitProvider @@ -251,6 +274,7 @@ class RemoteRepoSchema(BaseModel): # #region RemoteRepoCreateRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Provider-agnostic repository creation request. class RemoteRepoCreateRequest(BaseModel): name: str = Field(..., min_length=1, max_length=255) @@ -262,6 +286,7 @@ class RemoteRepoCreateRequest(BaseModel): # #region PromoteRequest [TYPE Class] +# @defgroup Api Module group. # @BRIEF Request schema for branch promotion workflow. class PromoteRequest(BaseModel): from_branch: str = Field(..., min_length=1, max_length=255) @@ -276,6 +301,7 @@ class PromoteRequest(BaseModel): # #region PromoteResponse [TYPE Class] +# @defgroup Api Module group. # @BRIEF Response schema for promotion operation result. class PromoteResponse(BaseModel): mode: str diff --git a/backend/src/api/routes/health.py b/backend/src/api/routes/health.py index 6196a2cd..f4d03bf3 100644 --- a/backend/src/api/routes/health.py +++ b/backend/src/api/routes/health.py @@ -1,4 +1,5 @@ # #region health_router [C:3] [TYPE Module] [SEMANTICS fastapi, health, api, search, dashboard] +# @defgroup Api Module group. # @BRIEF API endpoints for dashboard health monitoring and status aggregation. # @LAYER UI # @RELATION DEPENDS_ON -> [health_service] @@ -15,6 +16,7 @@ from ...services.health_service import HealthService router = APIRouter(prefix="/api/health", tags=["Health"]) # #region get_health_summary [TYPE Function] +# @ingroup Api # @BRIEF Get aggregated health status for all dashboards. # @PRE Caller has read permission for dashboard health view. # @POST Returns HealthSummaryResponse. @@ -38,6 +40,7 @@ async def get_health_summary( # #region delete_health_report [TYPE Function] +# @ingroup Api # @BRIEF Delete one persisted dashboard validation report from health summary. # @PRE Caller has write permission for tasks/report maintenance. # @POST Validation record is removed; linked task/logs are cleaned when available. diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 12152292..002d95e3 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -1,4 +1,5 @@ # #region LlmRoutes [C:3] [TYPE Module] [SEMANTICS fastapi, llm, api, provider] +# @defgroup Api Module group. # @BRIEF API routes for LLM provider configuration and management. # @LAYER API # @RELATION DEPENDS_ON -> [LLMProviderService] @@ -56,6 +57,7 @@ def _is_valid_runtime_api_key(value: str | None) -> bool: # #region get_providers [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Retrieve all LLM provider configurations. # @PRE User is authenticated. # @POST Returns list of LLMProviderConfig. @@ -96,6 +98,7 @@ async def get_providers( # #region fetch_models [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Fetch available models from an LLM provider by base_url+provider_type, or by provider_id. # @PRE User is authenticated. Either provider_id or base_url+provider_type must be provided. # @POST Returns a list of available model IDs. @@ -175,6 +178,7 @@ async def fetch_models( # #region get_llm_status [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Returns whether LLM runtime is configured for dashboard validation. # @PRE User is authenticated. # @POST configured=true only when an active provider with valid decrypted key exists. @@ -246,6 +250,7 @@ async def get_llm_status( # #region create_provider [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Create a new LLM provider configuration. # @PRE User is authenticated and has admin permissions. # @POST Returns the created LLMProviderConfig. @@ -283,6 +288,7 @@ async def create_provider( # #region update_provider [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Update an existing LLM provider configuration. # @PRE User is authenticated and has admin permissions. # @POST Returns the updated LLMProviderConfig. @@ -322,6 +328,7 @@ async def update_provider( # #region delete_provider [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Delete an LLM provider configuration. # @PRE User is authenticated and has admin permissions. # @POST Returns success status. @@ -360,6 +367,7 @@ async def delete_provider( # #region test_connection [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Test connection to an LLM provider. # @PRE User is authenticated. # @POST Returns success status and message. @@ -415,6 +423,7 @@ async def test_connection( # #region test_provider_config [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Test connection with a provided configuration (not yet saved). # @PRE User is authenticated. # @POST Returns success status and message. @@ -466,6 +475,7 @@ class ProbeMaxImagesResponse(BaseModel): # #region probe_max_images [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Probe an LLM provider to discover its max images per request limit. # @PRE Provider exists and has valid API key. # @POST Returns the detected max_images limit and caches it on the provider. diff --git a/backend/src/api/routes/maintenance/_router.py b/backend/src/api/routes/maintenance/_router.py index 8077b672..07e58c59 100644 --- a/backend/src/api/routes/maintenance/_router.py +++ b/backend/src/api/routes/maintenance/_router.py @@ -1,4 +1,5 @@ # #region MaintenanceRouter [C:3] [TYPE Module] [SEMANTICS fastapi, router, maintenance, api] +# @defgroup Api Module group. # @BRIEF FastAPI APIRouter for the Maintenance Banner endpoints. # @LAYER API # @RELATION DEPENDS_ON -> [MaintenanceRoutesModule] diff --git a/backend/src/api/routes/maintenance/_routes.py b/backend/src/api/routes/maintenance/_routes.py index 84d8808e..2311e398 100644 --- a/backend/src/api/routes/maintenance/_routes.py +++ b/backend/src/api/routes/maintenance/_routes.py @@ -1,4 +1,5 @@ # #region MaintenanceRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, routes, maintenance, api] +# @defgroup Api Module group. # @BRIEF Route handler stubs for all 7 Maintenance Banner API endpoints with RBAC guards per FR-015. # @LAYER API # @RELATION DEPENDS_ON -> [MaintenanceSchemasModule] @@ -52,6 +53,7 @@ from ._schemas import ( # FR-015: viewer, operator, admin all have READ access # #region list_dashboard_banners [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get per-dashboard banner state for the Dashboard Hub indicator. # @RELATION DEPENDS_ON -> [EXT:code:has_permission_maintenance_READ] @router.get("/dashboard-banners", response_model=list[MaintenanceDashboardBannerState]) @@ -116,6 +118,7 @@ async def list_dashboard_banners( # FR-015: operator, admin (NOT viewer) # #region list_events [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get active and completed maintenance event lists with affected dashboard counts. # @RELATION DEPENDS_ON -> [EXT:code:has_permission_maintenance_READ] @router.get("/events", response_model=MaintenanceEventListResponse) @@ -221,6 +224,7 @@ async def list_events( # FR-015: operator, admin (NOT viewer) # #region start_maintenance [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Start a maintenance event. Validates input, creates event, dispatches TaskManager task. # Always returns 202 with task_id and maintenance_id. # Idempotency: same (tables, start_time, end_time) returns already_active (409-like but 200). @@ -374,6 +378,7 @@ async def start_maintenance( # FR-015: operator, admin (NOT viewer) # #region end_maintenance [C:3] [TYPE Function] +# @ingroup Api # @BRIEF End a specific maintenance event by ID. Removes banners from affected dashboards. # Always returns 202 with task_id. Idempotent: already_completed returns success. # @RELATION DEPENDS_ON -> [EXT:code:has_permission("maintenance", "WRITE")] @@ -450,6 +455,7 @@ async def end_maintenance( # FR-015: operator, admin (NOT viewer) # #region end_all_maintenance [C:3] [TYPE Function] +# @ingroup Api # @BRIEF End all active maintenance events. Removes all banners from all dashboards. # Always returns 202 with task_id. Supports environment scoping for API keys. # @RELATION DEPENDS_ON -> [EXT:code:has_permission("maintenance", "WRITE")] @@ -520,6 +526,7 @@ async def end_all_maintenance( # FR-015: operator, admin (NOT viewer) # #region get_maintenance_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get current maintenance settings configuration. # @RELATION DEPENDS_ON -> [EXT:code:has_permission_maintenance_READ] @router.get("/settings", response_model=MaintenanceSettingsResponse) @@ -555,6 +562,7 @@ async def get_maintenance_settings( # FR-015: admin ONLY (NOT operator, NOT viewer) # #region update_maintenance_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Update maintenance settings. All fields optional for partial update. # admin role required per FR-015. # @RELATION DEPENDS_ON -> [EXT:code:has_permission("admin:settings", "WRITE")] diff --git a/backend/src/api/routes/maintenance/_schemas.py b/backend/src/api/routes/maintenance/_schemas.py index 1b5a01f4..c900ee15 100644 --- a/backend/src/api/routes/maintenance/_schemas.py +++ b/backend/src/api/routes/maintenance/_schemas.py @@ -1,4 +1,5 @@ # #region MaintenanceSchemasModule [C:2] [TYPE Module] [SEMANTICS pydantic, schema, maintenance, request, response] +# @defgroup Api Module group. # @BRIEF Pydantic models for Maintenance Banner API: request/response schemas per data-model.md. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] diff --git a/backend/src/api/routes/mappings.py b/backend/src/api/routes/mappings.py index f3ae057f..7a30aa32 100644 --- a/backend/src/api/routes/mappings.py +++ b/backend/src/api/routes/mappings.py @@ -1,4 +1,5 @@ # #region MappingsApi [C:3] [TYPE Module] [SEMANTICS fastapi, mapping, api, mapping-create] +# @defgroup Api Module group. # # @BRIEF API endpoints for managing database mappings and getting suggestions. # @LAYER API @@ -21,6 +22,7 @@ from ...models.mapping import DatabaseMapping router = APIRouter(tags=["mappings"]) # #region MappingCreate [TYPE DataClass] +# @ingroup Api class MappingCreate(BaseModel): source_env_id: str target_env_id: str @@ -32,6 +34,7 @@ class MappingCreate(BaseModel): # #endregion MappingCreate # #region MappingResponse [TYPE DataClass] +# @ingroup Api class MappingResponse(BaseModel): id: str source_env_id: str @@ -46,12 +49,14 @@ class MappingResponse(BaseModel): # #endregion MappingResponse # #region SuggestRequest [TYPE DataClass] +# @ingroup Api class SuggestRequest(BaseModel): source_env_id: str target_env_id: str # #endregion SuggestRequest # #region get_mappings [TYPE Function] +# @ingroup Api # @BRIEF List all saved database mappings. # @PRE db session is injected. # @POST Returns filtered list of DatabaseMapping records. @@ -72,6 +77,7 @@ async def get_mappings( # #endregion get_mappings # #region create_mapping [TYPE Function] +# @ingroup Api # @BRIEF Create or update a database mapping. # @PRE mapping is valid MappingCreate, db session is injected. # @POST DatabaseMapping created or updated in database. @@ -105,6 +111,7 @@ async def create_mapping( # #endregion create_mapping # #region suggest_mappings_api [TYPE Function] +# @ingroup Api # @BRIEF Get suggested mappings based on fuzzy matching. # @PRE request is valid SuggestRequest, config_manager is injected. # @POST Returns mapping suggestions. diff --git a/backend/src/api/routes/migration.py b/backend/src/api/routes/migration.py index 58fcd9cf..0bf36175 100644 --- a/backend/src/api/routes/migration.py +++ b/backend/src/api/routes/migration.py @@ -1,4 +1,5 @@ # #region MigrationApi [C:5] [TYPE Module] [SEMANTICS fastapi, migration, api, sync, search, mapping] +# @defgroup Api Module group. # @BRIEF HTTP contract layer for migration orchestration, settings, dry-run, and mapping sync endpoints. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [AppDependencies] @@ -41,6 +42,7 @@ router = APIRouter(prefix="/api", tags=["migration"]) # #region get_dashboards [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Fetch dashboard metadata from a requested environment for migration selection UI. # @PRE env_id is provided and exists in configured environments. # @POST Returns List[DashboardMetadata] for the resolved environment; emits HTTP_404 when environment is absent. @@ -72,6 +74,7 @@ async def get_dashboards( # #region execute_migration [C:5] [TYPE Function] +# @ingroup Api # @BRIEF Validate migration selection and enqueue asynchronous migration task execution. # @PRE DashboardSelection payload is valid and both source/target environments exist. # @POST Returns {"task_id": str, "message": str} when task creation succeeds; emits HTTP_400/HTTP_500 on failure. @@ -135,6 +138,7 @@ async def execute_migration( # #region dry_run_migration [C:5] [TYPE Function] +# @ingroup Api # @BRIEF Build pre-flight migration diff and risk summary without mutating target systems. # @PRE DashboardSelection is valid, source and target environments exist, differ, and selected_ids is non-empty. # @POST Returns deterministic dry-run payload; emits HTTP_400 for guard violations and HTTP_500 for orchestrator value errors. @@ -201,6 +205,7 @@ async def dry_run_migration( # #region get_migration_settings [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Read and return configured migration synchronization cron expression. # @PRE Configuration store is available and requester has READ permission. # @POST Returns {"cron": str} reflecting current persisted settings value. @@ -222,6 +227,7 @@ async def get_migration_settings( # #region update_migration_settings [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Validate and persist migration synchronization cron expression update. # @PRE Payload includes "cron" key and requester has WRITE permission. # @POST Returns {"cron": str, "status": "updated"} and persists updated cron value. @@ -253,6 +259,7 @@ async def update_migration_settings( # #region get_resource_mappings [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Fetch synchronized resource mappings with optional filters and pagination for migration mappings view. # @PRE skip>=0, 1<=limit<=500, DB session is active, requester has READ permission. # @POST Returns {"items": [...], "total": int} where items reflect applied filters and pagination. @@ -325,6 +332,7 @@ async def get_resource_mappings( # #region trigger_sync_now [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Trigger immediate ID synchronization for every configured environment. # @PRE At least one environment is configured and requester has EXECUTE permission. # @POST Returns sync summary with synced/failed counts after attempting all environments. diff --git a/backend/src/api/routes/plugins.py b/backend/src/api/routes/plugins.py index 26f811e2..40757b5f 100755 --- a/backend/src/api/routes/plugins.py +++ b/backend/src/api/routes/plugins.py @@ -1,4 +1,5 @@ # #region PluginsRouter [C:3] [TYPE Module] [SEMANTICS fastapi, plugin, api] +# @defgroup Api Module group. # @BRIEF Defines the FastAPI router for plugin-related endpoints, allowing clients to list available plugins. # @LAYER API # @RELATION DEPENDS_ON -> [PluginConfig] @@ -15,6 +16,7 @@ router = APIRouter() # #region list_plugins [TYPE Function] +# @ingroup Api # @BRIEF Retrieve a list of all available plugins. # @PRE plugin_loader is injected via Depends. # @POST Returns a list of PluginConfig objects. diff --git a/backend/src/api/routes/profile.py b/backend/src/api/routes/profile.py index eaa7132b..f112f080 100644 --- a/backend/src/api/routes/profile.py +++ b/backend/src/api/routes/profile.py @@ -1,4 +1,5 @@ # #region ProfileApiModule [C:5] [TYPE Module] [SEMANTICS fastapi, profile, api, validate, search, superset] +# @defgroup Api Module group. # # @BRIEF Exposes self-scoped profile preference endpoints and environment-based Superset account lookup. # @LAYER API @@ -60,6 +61,7 @@ def _get_profile_service(db: Session, config_manager, plugin_loader=None) -> Pro # #region get_preferences [TYPE Function] +# @ingroup Api # @RELATION CALLS -> ProfileService # @BRIEF Get authenticated user's dashboard filter preference. # @PRE Valid JWT and authenticated user context. @@ -79,6 +81,7 @@ async def get_preferences( # #region update_preferences [TYPE Function] +# @ingroup Api # @RELATION CALLS -> ProfileService # @BRIEF Update authenticated user's dashboard filter preference. # @PRE Valid JWT and valid request payload. @@ -106,6 +109,7 @@ async def update_preferences( # #region lookup_superset_accounts [TYPE Function] +# @ingroup Api # @RELATION CALLS -> ProfileService # @BRIEF Lookup Superset account candidates in selected environment. # @PRE Valid JWT, authenticated context, and environment_id query parameter. diff --git a/backend/src/api/routes/reports.py b/backend/src/api/routes/reports.py index 993da4a1..f37d74d0 100644 --- a/backend/src/api/routes/reports.py +++ b/backend/src/api/routes/reports.py @@ -1,4 +1,5 @@ # #region ReportsRouter [C:5] [TYPE Module] [SEMANTICS fastapi, report, api, transform, search, task] +# @defgroup Api Module group. # @BRIEF FastAPI router for unified task report list and detail retrieval endpoints. # @LAYER API # @RELATION DEPENDS_ON -> [ReportsService] @@ -69,6 +70,7 @@ def _parse_csv_enum_list(raw: str | None, enum_cls, field_name: str) -> list: # #region list_reports [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Return paginated unified reports list. # @PRE authenticated/authorized request and validated query params. # @POST returns {items,total,page,page_size,has_next,applied_filters}. @@ -144,6 +146,7 @@ async def list_reports( # #region get_report_detail [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Return one normalized report detail with diagnostics and next actions. # @PRE authenticated/authorized request and existing report_id. # @POST returns normalized detail envelope or 404 when report is not found. diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index f929c510..d71737ef 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -1,4 +1,5 @@ # #region SettingsRouter [C:5] [TYPE Module] [SEMANTICS fastapi, api, superset, logging-config-response] +# @defgroup Api Module group. # # @BRIEF Provides API endpoints for managing application settings and Superset environments. # @LAYER API @@ -91,6 +92,7 @@ async def _validate_superset_connection_fast(env: Environment) -> None: # #region get_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Retrieves all application settings. # @PRE Config manager is available. # @POST Returns masked AppConfig. @@ -145,6 +147,7 @@ async def get_allowed_languages( # #region update_global_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Updates global application settings. # @PRE New settings are provided. # @POST Global settings are updated. @@ -165,6 +168,7 @@ async def update_global_settings( # #region get_storage_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Retrieves storage-specific settings. @router.get("/storage", response_model=StorageConfig) async def get_storage_settings( @@ -179,6 +183,7 @@ async def get_storage_settings( # #region update_storage_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Updates storage-specific settings. # @POST Storage settings are updated and saved. @router.put("/storage", response_model=StorageConfig) @@ -202,6 +207,7 @@ async def update_storage_settings( # #region get_environments [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Lists all configured Superset environments. # @PRE Config manager is available. # @POST Returns list of environments. @@ -223,6 +229,7 @@ async def get_environments( # #region add_environment [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Adds a new Superset environment. # @PRE Environment data is valid and reachable. # @POST Environment is added to config. @@ -255,6 +262,7 @@ async def add_environment( # #region update_environment [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Updates an existing Superset environment. # @PRE ID and valid environment data are provided. # @POST Environment is updated in config. @@ -298,6 +306,7 @@ async def update_environment( # #region delete_environment [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Deletes a Superset environment. # @PRE ID is provided. # @POST Environment is removed from config. @@ -315,6 +324,7 @@ async def delete_environment( # #region test_environment_connection [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Tests the connection to a Superset environment. # @PRE ID is provided. # @POST Returns success or error status. @@ -348,6 +358,7 @@ async def test_environment_connection( # #region get_logging_config [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Retrieves current logging configuration. # @PRE Config manager is available. # @POST Returns logging configuration. @@ -369,6 +380,7 @@ async def get_logging_config( # #region update_logging_config [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Updates logging configuration. # @PRE New logging config is provided. # @POST Logging configuration is updated and saved. @@ -417,6 +429,7 @@ class ConsolidatedSettingsResponse(BaseModel): # #region get_consolidated_settings [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Retrieves all settings categories in a single call. # @PRE Config manager is available and the caller holds admin settings read permission. # @POST Returns consolidated settings, provider metadata, and persisted notification payload in one stable response. @@ -500,6 +513,7 @@ async def get_consolidated_settings( # #region update_consolidated_settings [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Bulk update application settings from the consolidated view. # @PRE User has admin permissions, config is valid. # @POST Settings are updated and saved via ConfigManager. @@ -569,6 +583,7 @@ async def update_consolidated_settings( # #region get_validation_policies [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Lists all validation policies. @router.get("/automation/policies", response_model=list[ValidationPolicyResponse]) async def get_validation_policies( @@ -582,6 +597,7 @@ async def get_validation_policies( # #region create_validation_policy [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Creates a new validation policy. @router.post("/automation/policies", response_model=ValidationPolicyResponse) async def create_validation_policy( @@ -601,6 +617,7 @@ async def create_validation_policy( # #region update_validation_policy [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Updates an existing validation policy. @router.patch("/automation/policies/{id}", response_model=ValidationPolicyResponse) async def update_validation_policy( @@ -627,6 +644,7 @@ async def update_validation_policy( # #region delete_validation_policy [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Deletes a validation policy. @router.delete("/automation/policies/{id}") async def delete_validation_policy( @@ -648,6 +666,7 @@ async def delete_validation_policy( # #region get_translation_schedules [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Lists all translation schedules joined with translation job names. @router.get("/automation/translation-schedules", response_model=list[TranslationScheduleItem]) async def get_translation_schedules( diff --git a/backend/src/api/routes/storage.py b/backend/src/api/routes/storage.py index fc63451d..debc4423 100644 --- a/backend/src/api/routes/storage.py +++ b/backend/src/api/routes/storage.py @@ -1,4 +1,5 @@ # #region storage_routes [C:5] [TYPE Module] [SEMANTICS fastapi, storage, api, upload, download] +# @defgroup Api Module group. # # @BRIEF API endpoints for file storage management (backups and repositories). # @LAYER API @@ -20,6 +21,7 @@ from ...plugins.storage.plugin import StoragePlugin router = APIRouter(tags=["storage"]) # #region list_files [C:3] [TYPE Function] +# @ingroup Api # @BRIEF List all files and directories in the storage system. # # @PRE None. @@ -42,6 +44,7 @@ async def list_files( # #endregion list_files # #region upload_file [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Upload a file to the storage system. # # @PRE category must be a valid FileCategory. @@ -71,6 +74,7 @@ async def upload_file( # #endregion upload_file # #region delete_file [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Delete a specific file or directory. # # @PRE category must be a valid FileCategory. @@ -100,6 +104,7 @@ async def delete_file( # #endregion delete_file # #region download_file [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Retrieve a file for download. # # @PRE category must be a valid FileCategory. @@ -129,6 +134,7 @@ async def download_file( # #endregion download_file # #region get_file_by_path [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Retrieve a file by validated absolute/relative path under storage root. # # @PRE path must resolve under configured storage root. diff --git a/backend/src/api/routes/tasks.py b/backend/src/api/routes/tasks.py index ac95e0d7..2fc8ea40 100755 --- a/backend/src/api/routes/tasks.py +++ b/backend/src/api/routes/tasks.py @@ -1,4 +1,5 @@ # #region TasksRouter [C:3] [TYPE Module] [SEMANTICS fastapi, task, api, search, create-task-request, resolve-task-request] +# @defgroup Api Module group. # @BRIEF Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks. # @LAYER API # @RELATION DEPENDS_ON -> [TaskManager] @@ -48,6 +49,7 @@ class ResumeTaskRequest(BaseModel): # #region create_task [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Create and start a new task for a given plugin. # @PRE plugin_id must exist and params must be valid for that plugin. # @POST A new task is created and started. @@ -132,6 +134,7 @@ async def create_task( # #region list_tasks [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Retrieve a list of tasks with pagination and optional status filter. # @PRE task_manager must be available. # @POST Returns a list of tasks. @@ -177,6 +180,7 @@ async def list_tasks( # #region get_task [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Retrieve the details of a specific task. # @PRE task_id must exist. # @POST Returns task details or raises 404. @@ -200,6 +204,7 @@ async def get_task( # #region get_task_logs [C:5] [TYPE Function] +# @ingroup Api # @BRIEF Retrieve logs for a specific task with optional filtering. # @PRE task_id must exist. # @POST Returns a list of log entries or raises 404. @@ -247,6 +252,7 @@ async def get_task_logs( # #region get_task_log_stats [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get statistics about logs for a task (counts by level and source). # @PRE task_id must exist. # @POST Returns log statistics or raises 404. @@ -292,6 +298,7 @@ async def get_task_log_stats( # #region get_task_log_sources [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Get unique sources for a task's logs. # @PRE task_id must exist. # @POST Returns list of unique source names or raises 404. @@ -314,6 +321,7 @@ async def get_task_log_sources( # #region resolve_task [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Resolve a task that is awaiting mapping. # @PRE task must be in AWAITING_MAPPING status. # @POST Task is resolved and resumes execution. @@ -337,6 +345,7 @@ async def resolve_task( # #region resume_task [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Resume a task that is awaiting input (e.g., passwords). # @PRE task must be in AWAITING_INPUT status. # @POST Task resumes execution with provided input. @@ -360,6 +369,7 @@ async def resume_task( # #region clear_tasks [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Clear tasks matching the status filter. # @PRE task_manager is available. # @POST Tasks are removed from memory/persistence. diff --git a/backend/src/api/routes/translate/__init__.py b/backend/src/api/routes/translate/__init__.py index 16f440d4..c1c7b80d 100644 --- a/backend/src/api/routes/translate/__init__.py +++ b/backend/src/api/routes/translate/__init__.py @@ -1,4 +1,5 @@ # #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS translate, api, package, schedule, dashboard, llm] +# @defgroup Api Module group. # @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import. # @LAYER API # @RELATION DEPENDS_ON -> [SupersetClient] diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py index 1e7ddd22..9eddca3b 100644 --- a/backend/src/api/routes/translate/_correction_routes.py +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -1,4 +1,5 @@ # #region TranslateCorrectionRoutesModule [C:2] [TYPE Module] [SEMANTICS fastapi, translate, api, search] +# @defgroup Api Module group. # @BRIEF Term correction submission endpoints. # @LAYER API @@ -21,6 +22,7 @@ from ._router import router # ============================================================ # #region submit_correction [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Submit a term correction from a run result review. # @PRE User has translate.dictionary.edit permission. # @POST Correction applied with conflict detection. @@ -58,6 +60,7 @@ async def submit_correction( # #region submit_bulk_corrections [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Submit multiple term corrections atomically. # @PRE User has translate.dictionary.edit permission. # @POST All corrections applied or none with conflict list. diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py index 11425fdf..50419412 100644 --- a/backend/src/api/routes/translate/_dictionary_routes.py +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -1,4 +1,5 @@ # #region TranslateDictionaryRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, search] +# @defgroup Api Module group. # @BRIEF Terminology Dictionary CRUD, entries management, and import routes. # @LAYER API @@ -23,6 +24,7 @@ from ._router import router # ============================================================ # #region list_dictionaries [C:4] [TYPE Function] +# @ingroup Api # @BRIEF List all terminology dictionaries. # @PRE User has translate.dictionary.view permission. # @POST Returns list of dictionaries with total count. @@ -47,6 +49,7 @@ async def list_dictionaries( # #region get_dictionary [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get a single terminology dictionary by ID. # @PRE User has translate.dictionary.view permission. # @POST Returns the dictionary with entry_count. @@ -72,6 +75,7 @@ async def get_dictionary( # #region create_dictionary [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Create a new terminology dictionary. # @PRE User has translate.dictionary.create permission. # @POST Returns the created dictionary. @@ -98,6 +102,7 @@ async def create_dictionary( # #region update_dictionary [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Update an existing terminology dictionary. # @PRE User has translate.dictionary.edit permission. # @POST Returns the updated dictionary. @@ -130,6 +135,7 @@ async def update_dictionary( # #region delete_dictionary [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs. # @PRE User has translate.dictionary.delete permission. # @POST Dictionary is deleted. @@ -159,6 +165,7 @@ async def delete_dictionary( # ============================================================ # #region list_dictionary_entries [C:4] [TYPE Function] +# @ingroup Api # @BRIEF List entries for a dictionary, optionally filtered by language pair. # @PRE User has translate.dictionary.view permission. # @POST Returns paginated list of entries with language pair fields. @@ -218,6 +225,7 @@ async def list_dictionary_entries( # #region add_dictionary_entry [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Add a new entry to a dictionary. # @PRE User has translate.dictionary.edit permission. # @POST Entry is created. @@ -269,6 +277,7 @@ async def add_dictionary_entry( # #region edit_dictionary_entry [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Update an existing dictionary entry. # @PRE User has translate.dictionary.edit permission. # @POST Entry is updated. @@ -321,6 +330,7 @@ async def edit_dictionary_entry( # #region delete_dictionary_entry [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Delete a dictionary entry. # @PRE User has translate.dictionary.edit permission. # @POST Entry is deleted. @@ -345,6 +355,7 @@ async def delete_dictionary_entry( # #region import_dictionary_entries [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Import entries into a terminology dictionary from CSV/TSV content. # @PRE User has translate.dictionary.edit permission. # @POST Entries are imported per conflict mode. diff --git a/backend/src/api/routes/translate/_helpers.py b/backend/src/api/routes/translate/_helpers.py index a99a968e..c184b3eb 100644 --- a/backend/src/api/routes/translate/_helpers.py +++ b/backend/src/api/routes/translate/_helpers.py @@ -1,4 +1,5 @@ # #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS sqlalchemy, translate, helper, query, mapping] +# @defgroup Api Module group. # @BRIEF Shared helper functions for translate route handlers. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:frontend:models.translate] diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py index 2202c685..6c221fda 100644 --- a/backend/src/api/routes/translate/_job_routes.py +++ b/backend/src/api/routes/translate/_job_routes.py @@ -1,4 +1,5 @@ # #region TranslateJobRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, search] +# @defgroup Api Module group. # @BRIEF Translation Job CRUD and datasource column routes. # @LAYER API @@ -33,6 +34,7 @@ from ._router import router # ============================================================ # #region list_jobs [C:4] [TYPE Function] +# @ingroup Api # @BRIEF List all translation jobs. # @PRE User has translate.job.view permission. # @POST Returns list of translation jobs. @@ -66,6 +68,7 @@ async def list_jobs( # #region get_job [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get a single translation job by ID. # @PRE User has translate.job.view permission. # @POST Returns the translation job. @@ -90,6 +93,7 @@ async def get_job( # #region create_job [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Create a new translation job. # @PRE User has translate.job.create permission. # @POST Returns the created translation job. @@ -115,6 +119,7 @@ async def create_job( # #region update_job [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Update an existing translation job. # @PRE User has translate.job.edit permission. # @POST Returns the updated translation job. @@ -141,6 +146,7 @@ async def update_job( # #region delete_job [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Delete a translation job. # @PRE User has translate.job.delete permission. # @POST Job is deleted. @@ -163,6 +169,7 @@ async def delete_job( # #region duplicate_job [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Duplicate a translation job. # @PRE User has translate.job.create permission. # @POST Returns the duplicated job with status DRAFT. @@ -190,6 +197,7 @@ async def duplicate_job( # #region get_datasource_columns [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get column metadata and database dialect for a Superset datasource. # @PRE User has translate.job.view permission. # @POST Returns column list with metadata and database_dialect. @@ -249,6 +257,7 @@ async def get_datasource_columns( # #region check_target_schema [C:3] [TYPE Function] [SEMANTICS api,translate,validate] +# @ingroup Api # @BRIEF Проверяет схему целевой таблицы: какие колонки ожидаются, какие есть, каких не хватает. # @RELATION DEPENDS_ON -> [validate_target_table_schema] # @RELATION DEPENDS_ON -> [TargetSchemaValidationRequest] diff --git a/backend/src/api/routes/translate/_metrics_routes.py b/backend/src/api/routes/translate/_metrics_routes.py index 301e09e8..cddf8e45 100644 --- a/backend/src/api/routes/translate/_metrics_routes.py +++ b/backend/src/api/routes/translate/_metrics_routes.py @@ -1,4 +1,5 @@ # #region TranslateMetricsRoutesModule [C:2] [TYPE Module] [SEMANTICS fastapi, translate, api, search] +# @defgroup Api Module group. # @BRIEF Translation Metrics endpoints. # @LAYER API @@ -18,6 +19,7 @@ from ._router import router # ============================================================ # #region get_metrics [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get translation metrics, optionally filtered by job. # @PRE User has translate.metrics.view permission. # @POST Returns metrics data. @@ -41,6 +43,7 @@ async def get_metrics( # #region get_job_metrics [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get aggregated metrics for a specific job. # @PRE User has translate.metrics.view permission. # @POST Returns metrics dict. diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py index 99ad96b4..27ed2c58 100644 --- a/backend/src/api/routes/translate/_preview_routes.py +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -1,4 +1,5 @@ # #region TranslatePreviewRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, preview, review, api, search] +# @defgroup Api Module group. # @BRIEF Translation Preview session management routes. # @LAYER API @@ -24,6 +25,7 @@ from ._router import router # ============================================================ # #region preview_translation [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Preview a translation before applying it. # @PRE User has translate.job.execute permission. # @POST Returns a preview session with records and cost estimation. @@ -60,6 +62,7 @@ async def preview_translation( # #region update_preview_row [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Approve, edit, or reject a preview row (optionally per language). # @PRE User has translate.job.execute permission. # @POST Preview row status is updated. @@ -92,6 +95,7 @@ async def update_preview_row( # #region accept_preview_session [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Accept a preview session, marking it as the quality gate for full execution. # @PRE User has translate.job.execute permission. Job has an ACTIVE preview session. # @POST Preview session is marked as APPLIED; full execution can proceed. @@ -115,6 +119,7 @@ async def accept_preview_session( # #region apply_preview [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Apply a preview session (alias for accept when accepting at session level). # @PRE User has translate.job.execute permission. # @POST Preview is applied. @@ -147,6 +152,7 @@ async def apply_preview( # ============================================================ # #region get_preview_records [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get records for a preview session. # @PRE User has translate.job.view permission. # @POST Returns list of preview records. diff --git a/backend/src/api/routes/translate/_router.py b/backend/src/api/routes/translate/_router.py index 945f82cc..5c28e1f9 100644 --- a/backend/src/api/routes/translate/_router.py +++ b/backend/src/api/routes/translate/_router.py @@ -5,6 +5,7 @@ from fastapi import APIRouter # #region translate_router [TYPE Global] +# @ingroup Api # @BRIEF APIRouter instance for all translate sub-routes. router = APIRouter(prefix="/api/translate", tags=["translate"]) diff --git a/backend/src/api/routes/translate/_run_edit_routes.py b/backend/src/api/routes/translate/_run_edit_routes.py index c0147e4d..272ef627 100644 --- a/backend/src/api/routes/translate/_run_edit_routes.py +++ b/backend/src/api/routes/translate/_run_edit_routes.py @@ -1,4 +1,5 @@ # #region TranslateRunEditRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, edit, correction, bulk, override] +# @defgroup Api Module group. # @BRIEF Translation Run inline edit, bulk find-replace, and language override routes. # @LAYER API # @RELATION DEPENDS_ON -> [TranslateRunRoutesModule] @@ -21,6 +22,7 @@ from ._router import router # #region override_detected_language [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Manually override the detected source language for a specific translation language entry. # @PRE User has translate.job.execute permission. Run, record, and language entry exist. # @POST TranslationLanguage.source_language_detected is updated; language_overridden is set to True. @@ -96,6 +98,7 @@ def override_detected_language( # #region inline_edit_translation [C:4] [TYPE Function] [SEMANTICS api,translate,correction] +# @ingroup Api # @BRIEF Apply an inline correction to a translated value on a completed run result. # @PRE User has translate.job.execute permission. Run, record, and language entry exist. # @POST TranslationLanguage.final_value and user_edit are updated. Optional dictionary submission. @@ -140,6 +143,7 @@ def inline_edit_translation( # #region bulk_find_replace [C:4] [TYPE Function] [SEMANTICS api,translate,bulk,replace] +# @ingroup Api # @BRIEF Perform bulk find-and-replace on translated values within a run. # @PRE User has translate.job.execute permission. Run exists. # @POST If preview=false, matching translations are updated. Optional dictionary submission. diff --git a/backend/src/api/routes/translate/_run_history_routes.py b/backend/src/api/routes/translate/_run_history_routes.py index 177308e8..6fc5b0eb 100644 --- a/backend/src/api/routes/translate/_run_history_routes.py +++ b/backend/src/api/routes/translate/_run_history_routes.py @@ -1,4 +1,5 @@ # #region TranslateRunHistoryRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, history, status, records] +# @defgroup Api Module group. # @BRIEF Translation Run history, status, records and batches routes. # @LAYER API # @RELATION DEPENDS_ON -> [TranslateRunRoutesModule] @@ -20,6 +21,7 @@ from datetime import datetime, UTC # #region get_run_history [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get run history for a translation job. # @PRE User has translate.history.view permission. # @POST Returns list of runs. @@ -44,6 +46,7 @@ def get_run_history( # #region get_run_status [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get status and statistics for a translation run. # @PRE User has translate.history.view permission. # @POST Returns run details with statistics. @@ -65,6 +68,7 @@ def get_run_status( # #region get_run_records [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get paginated records for a translation run. # @PRE User has translate.history.view permission. # @POST Returns paginated records. @@ -90,6 +94,7 @@ def get_run_records( # #region get_batches [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get batches for a translation run. # @PRE User has translate.job.view permission. # @POST Returns list of batches. diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index 558a7770..40aa68a7 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -1,4 +1,5 @@ # #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, download, search] +# @defgroup Api Module group. # @BRIEF Translation Run listing, detail, and CSV download routes (cross-job). # @LAYER API @@ -20,6 +21,7 @@ from ._router import router # ============================================================ # #region list_runs [C:4] [TYPE Function] +# @ingroup Api # @BRIEF List all runs with cross-job filtering and pagination. # @PRE User has translate.history.view permission. # @POST Returns paginated list of runs. @@ -128,6 +130,7 @@ async def list_runs( # ============================================================ # #region get_run_detail [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get detailed run info with config_snapshot, records, events. # @PRE User has translate.history.view permission. # @POST Returns run detail with records and events. @@ -171,6 +174,7 @@ async def get_run_detail( # ============================================================ # #region download_skipped_csv [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Download a CSV of skipped translation records from a run. # @PRE User has translate.job.view permission. # @POST Returns CSV file of skipped records. @@ -229,6 +233,7 @@ async def download_skipped_csv( # ============================================================ # #region download_failed_csv [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Download a CSV of successfully translated but uninserted records from a FAILED run. # @PRE User has translate.job.view permission. Run must have insert_status=failed or status=FAILED. # @POST Returns CSV file of records that were translated but not inserted into target DB. diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index c005917a..1d229cec 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -1,4 +1,5 @@ # #region TranslateRunRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, execution] +# @defgroup Api Module group. # @BRIEF Translation Run execution, retry, and cancel routes. # @LAYER API # @RELATION DEPENDS_ON -> [TranslateRunHistoryRoutesModule] @@ -23,6 +24,7 @@ from ._router import router # #region run_translation [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Execute a translation job (trigger a run). # @PRE User has translate.job.execute permission. # @POST Returns the created translation run. @@ -128,6 +130,7 @@ async def run_translation( # #region retry_run [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Retry failed batches in a translation run. # @PRE User has translate.job.execute permission. # @POST Returns the updated translation run. @@ -154,6 +157,7 @@ async def retry_run( # #region retry_insert [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Retry the SQL insert phase for a completed run. # @PRE User has translate.job.execute permission. # @POST Returns the updated run. @@ -180,6 +184,7 @@ async def retry_insert( # #region cancel_run [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Cancel a running translation. # @PRE User has translate.job.execute permission. # @POST Run is cancelled. diff --git a/backend/src/api/routes/translate/_schedule_routes.py b/backend/src/api/routes/translate/_schedule_routes.py index 25d22d28..51991bea 100644 --- a/backend/src/api/routes/translate/_schedule_routes.py +++ b/backend/src/api/routes/translate/_schedule_routes.py @@ -1,4 +1,5 @@ # #region TranslateScheduleRoutesModule [C:3] [TYPE Module] [SEMANTICS fastapi, translate, api, schedule, search] +# @defgroup Api Module group. # @BRIEF Translation Schedule management routes. # @LAYER API @@ -19,6 +20,7 @@ from ._router import router # ============================================================ # #region get_schedule [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Get the schedule for a translation job. # @PRE User has translate.schedule.view permission. # @POST Returns the schedule configuration. @@ -54,6 +56,7 @@ async def get_schedule( # #region set_schedule [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Set or update the schedule for a translation job. # @PRE User has translate.schedule.manage permission. # @POST Schedule is created or updated. @@ -120,6 +123,7 @@ async def set_schedule( # #region enable_schedule [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Enable a schedule for a translation job. # @PRE User has translate.schedule.manage permission. # @POST Schedule is enabled. @@ -152,6 +156,7 @@ async def enable_schedule( # #region disable_schedule [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Disable a schedule for a translation job. # @PRE User has translate.schedule.manage permission. # @POST Schedule is disabled. @@ -178,6 +183,7 @@ async def disable_schedule( # #region delete_schedule [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Delete the schedule for a translation job. # @PRE User has translate.schedule.manage permission. # @POST Schedule is removed. @@ -204,6 +210,7 @@ async def delete_schedule( # #region get_next_executions [C:4] [TYPE Function] +# @ingroup Api # @BRIEF Preview next N executions for a job's schedule. # @PRE User has translate.schedule.view permission. # @POST Returns next execution times. diff --git a/backend/src/api/routes/validation/__tests__/test_validation_api.py b/backend/src/api/routes/validation/__tests__/test_validation_api.py index b829aa49..31e7aca8 100644 --- a/backend/src/api/routes/validation/__tests__/test_validation_api.py +++ b/backend/src/api/routes/validation/__tests__/test_validation_api.py @@ -35,18 +35,19 @@ from unittest.mock import AsyncMock, MagicMock from fastapi.testclient import TestClient -from src.api.routes.validation import _get_run_service, _get_task_service +from src.api.routes.validation_tasks import _get_task_service from src.app import app from src.dependencies import ( get_config_manager, get_current_user, + get_scheduler_service, get_task_manager, ) from src.models.auth import User from src.schemas.validation import ( TriggerRunResponse, - ValidationRunDetailResponse, - ValidationRunListResponse, + RunDetailResponse, + RunListResponse, ValidationTaskListResponse, ValidationTaskResponse, ) @@ -78,19 +79,20 @@ def mock_all_deps(): # Reset user state — Admin role passes all permission gates mock_user.roles = [admin_role] - mock_task_service = MagicMock() - mock_run_service = MagicMock() + mock_validation_service = MagicMock() mock_task_manager = MagicMock() - app.dependency_overrides[_get_task_service] = lambda: mock_task_service - app.dependency_overrides[_get_run_service] = lambda: mock_run_service + mock_scheduler_service = MagicMock() + + app.dependency_overrides[_get_task_service] = lambda: mock_validation_service app.dependency_overrides[get_config_manager] = lambda: MagicMock() app.dependency_overrides[get_task_manager] = lambda: mock_task_manager + app.dependency_overrides[get_scheduler_service] = lambda: mock_scheduler_service app.dependency_overrides[get_current_user] = lambda: mock_user yield { - "task_service": mock_task_service, - "run_service": mock_run_service, + "task_service": mock_validation_service, + "run_service": mock_validation_service, "task_manager": mock_task_manager, } app.dependency_overrides.clear() @@ -146,7 +148,7 @@ def test_list_tasks_success(mock_all_deps): task = _make_task_response() mock_all_deps["task_service"].list_tasks.return_value = (1, [task]) - response = client.get("/api/validation/tasks") + response = client.get("/api/validation-tasks") assert response.status_code == 200 data = response.json() assert data["items"][0]["name"] == "Test Task" @@ -156,7 +158,7 @@ def test_list_tasks_success(mock_all_deps): ValidationTaskListResponse(**data) # environment_id filter - response = client.get("/api/validation/tasks?environment_id=env-1") + response = client.get("/api/validation-tasks?environment_id=env-1") assert response.status_code == 200 mock_all_deps["task_service"].list_tasks.assert_called_with( is_active=None, environment_id="env-1", page=1, page_size=20 @@ -167,13 +169,13 @@ def test_list_tasks_success(mock_all_deps): # #region test_list_tasks_pagination [C:2] [TYPE Function] # @BRIEF Pagination query params are validated: page >= 1, page_size 1-100. def test_list_tasks_pagination(mock_all_deps): - response = client.get("/api/validation/tasks?page=0") + response = client.get("/api/validation-tasks?page=0") assert response.status_code == 422 - response = client.get("/api/validation/tasks?page_size=101") + response = client.get("/api/validation-tasks?page_size=101") assert response.status_code == 422 - response = client.get("/api/validation/tasks?page_size=0") + response = client.get("/api/validation-tasks?page_size=0") assert response.status_code == 422 # #endregion test_list_tasks_pagination @@ -182,7 +184,7 @@ def test_list_tasks_pagination(mock_all_deps): # @BRIEF Service ValueError becomes 400 on list_tasks. def test_list_tasks_service_error(mock_all_deps): mock_all_deps["task_service"].list_tasks.side_effect = ValueError("Bad filter") - response = client.get("/api/validation/tasks?environment_id=nonexistent") + response = client.get("/api/validation-tasks?environment_id=nonexistent") assert response.status_code == 400 assert "Bad filter" in response.json()["detail"] # #endregion test_list_tasks_service_error @@ -201,7 +203,7 @@ def test_create_task_success(mock_all_deps): "notify_owners": True, "alert_condition": "FAIL_ONLY", } - response = client.post("/api/validation/tasks", json=payload) + response = client.post("/api/validation-tasks", json=payload) assert response.status_code == 201 data = response.json() assert data["name"] == "Test Task" @@ -215,19 +217,19 @@ def test_create_task_success(mock_all_deps): # @BRIEF POST with missing required fields returns 422. def test_create_task_missing_fields(mock_all_deps): # Empty payload - response = client.post("/api/validation/tasks", json={}) + response = client.post("/api/validation-tasks", json={}) assert response.status_code == 422 errors = response.json() assert "detail" in errors # Missing provider_id - response = client.post("/api/validation/tasks", json={ + response = client.post("/api/validation-tasks", json={ "name": "Test", "environment_id": "env-1", "dashboard_ids": ["dash-1"], }) assert response.status_code == 422 # Missing dashboard_ids - response = client.post("/api/validation/tasks", json={ + response = client.post("/api/validation-tasks", json={ "name": "Test", "environment_id": "env-1", "provider_id": "provider-1", }) assert response.status_code == 422 @@ -244,7 +246,7 @@ def test_create_task_invalid_provider(mock_all_deps): "name": "Test", "environment_id": "env-1", "dashboard_ids": ["dash-1"], "provider_id": "provider-2", } - response = client.post("/api/validation/tasks", json=payload) + response = client.post("/api/validation-tasks", json=payload) assert response.status_code == 422 assert "not multimodal" in response.json()["detail"] # #endregion test_create_task_invalid_provider @@ -256,7 +258,7 @@ def test_get_task_success(mock_all_deps): mock_all_deps["task_service"].get_task.return_value = _make_task_response() mock_all_deps["run_service"].list_runs.return_value = (0, []) - response = client.get("/api/validation/tasks/task-1") + response = client.get("/api/validation-tasks/task-1") assert response.status_code == 200 data = response.json() assert "task" in data @@ -270,7 +272,7 @@ def test_get_task_success(mock_all_deps): # @BRIEF GET for nonexistent task returns 404. def test_get_task_not_found(mock_all_deps): mock_all_deps["task_service"].get_task.side_effect = ValueError("not found") - response = client.get("/api/validation/tasks/nonexistent") + response = client.get("/api/validation-tasks/nonexistent") assert response.status_code == 404 # #endregion test_get_task_not_found @@ -282,7 +284,7 @@ def test_update_task_success(mock_all_deps): mock_all_deps["task_service"].update_task.return_value = updated payload = {"name": "Updated Name", "is_active": False} - response = client.put("/api/validation/tasks/task-1", json=payload) + response = client.put("/api/validation-tasks/task-1", json=payload) assert response.status_code == 200 data = response.json() assert data["name"] == "Updated Name" @@ -297,7 +299,7 @@ def test_update_task_success(mock_all_deps): def test_update_task_not_found(mock_all_deps): mock_all_deps["task_service"].update_task.side_effect = ValueError("not found") payload = {"name": "Nope"} - response = client.put("/api/validation/tasks/nonexistent", json=payload) + response = client.put("/api/validation-tasks/nonexistent", json=payload) assert response.status_code == 404 # #endregion test_update_task_not_found @@ -306,7 +308,7 @@ def test_update_task_not_found(mock_all_deps): # @BRIEF DELETE /validation/tasks/{id} returns 204. def test_delete_task_success(mock_all_deps): mock_all_deps["task_service"].delete_task.return_value = None - response = client.delete("/api/validation/tasks/task-1") + response = client.delete("/api/validation-tasks/task-1") assert response.status_code == 204 mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=False) # #endregion test_delete_task_success @@ -316,7 +318,7 @@ def test_delete_task_success(mock_all_deps): # @BRIEF DELETE for nonexistent task returns 404. def test_delete_task_not_found(mock_all_deps): mock_all_deps["task_service"].delete_task.side_effect = ValueError("not found") - response = client.delete("/api/validation/tasks/nonexistent") + response = client.delete("/api/validation-tasks/nonexistent") assert response.status_code == 404 # #endregion test_delete_task_not_found @@ -325,7 +327,7 @@ def test_delete_task_not_found(mock_all_deps): # @BRIEF DELETE with delete_runs=true passes query param to service. def test_delete_task_with_runs_flag(mock_all_deps): mock_all_deps["task_service"].delete_task.return_value = None - response = client.delete("/api/validation/tasks/task-1?delete_runs=true") + response = client.delete("/api/validation-tasks/task-1?delete_runs=true") assert response.status_code == 204 mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=True) # #endregion test_delete_task_with_runs_flag @@ -337,7 +339,7 @@ def test_trigger_run_success(mock_all_deps): mock_all_deps["task_service"].trigger_run = AsyncMock(return_value="spawned-1") mock_all_deps["task_manager"].create_task = AsyncMock() - response = client.post("/api/validation/tasks/task-1/run") + response = client.post("/api/validation-tasks/task-1/run") assert response.status_code == 200 data = response.json() assert data["task_id"] == "task-1" @@ -353,7 +355,7 @@ def test_trigger_run_not_found(mock_all_deps): mock_all_deps["task_service"].trigger_run = AsyncMock( side_effect=ValueError("Validation task 'nonexistent' not found") ) - response = client.post("/api/validation/tasks/nonexistent/run") + response = client.post("/api/validation-tasks/nonexistent/run") assert response.status_code == 422 assert "not found" in response.json()["detail"] # #endregion test_trigger_run_not_found @@ -365,7 +367,7 @@ def test_trigger_run_no_dashboards(mock_all_deps): mock_all_deps["task_service"].trigger_run = AsyncMock( side_effect=ValueError("has no dashboard IDs configured") ) - response = client.post("/api/validation/tasks/task-1/run") + response = client.post("/api/validation-tasks/task-1/run") assert response.status_code == 422 assert "no dashboard IDs" in response.json()["detail"] # #endregion test_trigger_run_no_dashboards @@ -379,12 +381,12 @@ def test_list_runs_success(mock_all_deps): run = _make_run_response() mock_all_deps["run_service"].list_runs.return_value = (1, [run]) - response = client.get("/api/validation/runs") + response = client.get("/api/validation-runs") assert response.status_code == 200 data = response.json() assert data["items"][0]["id"] == "run-1" assert data["total"] == 1 - ValidationRunListResponse(**data) + RunListResponse(**data) # #endregion test_list_runs_success @@ -402,7 +404,7 @@ def test_list_runs_filters(mock_all_deps): "?date_to=2026-12-31", "?status=PASS&dashboard_id=dash-1&date_from=2026-01-01", ]: - response = client.get(f"/api/validation/runs{query}") + response = client.get(f"/api/validation-runs{query}") assert response.status_code == 200, f"Filter query failed: {query}" # #endregion test_list_runs_filters @@ -410,10 +412,10 @@ def test_list_runs_filters(mock_all_deps): # #region test_list_runs_pagination [C:2] [TYPE Function] # @BRIEF Pagination boundary checks on runs list. def test_list_runs_pagination(mock_all_deps): - response = client.get("/api/validation/runs?page=0") + response = client.get("/api/validation-runs?page=0") assert response.status_code == 422 - response = client.get("/api/validation/runs?page_size=101") + response = client.get("/api/validation-runs?page_size=101") assert response.status_code == 422 # #endregion test_list_runs_pagination @@ -422,7 +424,7 @@ def test_list_runs_pagination(mock_all_deps): # @BRIEF Status is a free string; any value passes through to service. def test_list_runs_invalid_status(mock_all_deps): mock_all_deps["run_service"].list_runs.return_value = (0, []) - response = client.get("/api/validation/runs?status=INVALID_STATUS_XYZ") + response = client.get("/api/validation-runs?status=INVALID_STATUS_XYZ") assert response.status_code == 200 call_kwargs = mock_all_deps["run_service"].list_runs.call_args[1] assert call_kwargs.get("status") == "INVALID_STATUS_XYZ" @@ -432,7 +434,7 @@ def test_list_runs_invalid_status(mock_all_deps): # #region test_get_run_detail_success [C:2] [TYPE Function] # @BRIEF GET /validation/runs/{id} returns full detail with issues and raw_response. def test_get_run_detail_success(mock_all_deps): - mock_all_deps["run_service"].get_run_detail.return_value = ValidationRunDetailResponse( + mock_all_deps["run_service"].get_run_detail.return_value = RunDetailResponse( id="run-1", task_id="task-1", dashboard_id="dash-1", @@ -446,13 +448,13 @@ def test_get_run_detail_success(mock_all_deps): task_name="Test Task", ) - response = client.get("/api/validation/runs/run-1") + response = client.get("/api/validation-runs/run-1") assert response.status_code == 200 data = response.json() assert data["status"] == "FAIL" assert len(data["issues"]) == 1 assert data["raw_response"] is not None - ValidationRunDetailResponse(**data) + RunDetailResponse(**data) # #endregion test_get_run_detail_success @@ -460,7 +462,7 @@ def test_get_run_detail_success(mock_all_deps): # @BRIEF GET for nonexistent run returns 404. def test_get_run_detail_not_found(mock_all_deps): mock_all_deps["run_service"].get_run_detail.side_effect = ValueError("not found") - response = client.get("/api/validation/runs/nonexistent") + response = client.get("/api/validation-runs/nonexistent") assert response.status_code == 404 # #endregion test_get_run_detail_not_found @@ -469,7 +471,7 @@ def test_get_run_detail_not_found(mock_all_deps): # @BRIEF DELETE /validation/runs/{id} returns 204. def test_delete_run_success(mock_all_deps): mock_all_deps["run_service"].delete_run.return_value = True - response = client.delete("/api/validation/runs/run-1") + response = client.delete("/api/validation-runs/run-1") assert response.status_code == 204 mock_all_deps["run_service"].delete_run.assert_called_with("run-1") # #endregion test_delete_run_success @@ -479,7 +481,7 @@ def test_delete_run_success(mock_all_deps): # @BRIEF DELETE for nonexistent run returns 404. def test_delete_run_not_found(mock_all_deps): mock_all_deps["run_service"].delete_run.return_value = False - response = client.delete("/api/validation/runs/nonexistent") + response = client.delete("/api/validation-runs/nonexistent") assert response.status_code == 404 # #endregion test_delete_run_not_found @@ -491,15 +493,15 @@ def test_delete_run_not_found(mock_all_deps): def test_endpoints_require_authentication(mock_all_deps): app.dependency_overrides.pop(get_current_user, None) endpoints = [ - ("GET", "/api/validation/tasks"), - ("POST", "/api/validation/tasks"), - ("GET", "/api/validation/tasks/task-1"), - ("PUT", "/api/validation/tasks/task-1"), - ("DELETE", "/api/validation/tasks/task-1"), - ("POST", "/api/validation/tasks/task-1/run"), - ("GET", "/api/validation/runs"), - ("GET", "/api/validation/runs/run-1"), - ("DELETE", "/api/validation/runs/run-1"), + ("GET", "/api/validation-tasks"), + ("POST", "/api/validation-tasks"), + ("GET", "/api/validation-tasks/task-1"), + ("PUT", "/api/validation-tasks/task-1"), + ("DELETE", "/api/validation-tasks/task-1"), + ("POST", "/api/validation-tasks/task-1/run"), + ("GET", "/api/validation-runs"), + ("GET", "/api/validation-runs/run-1"), + ("DELETE", "/api/validation-runs/run-1"), ] for method, path in endpoints: response = client.request(method, path) @@ -512,7 +514,7 @@ def test_endpoints_require_authentication(mock_all_deps): def test_permission_denied_non_admin(mock_all_deps): """Remove Admin role so has_permission raises 403.""" mock_user.roles = [] # No Admin, no permissions - response = client.get("/api/validation/tasks") + response = client.get("/api/validation-tasks") assert response.status_code == 403 # #endregion test_permission_denied_non_admin @@ -522,15 +524,15 @@ def test_permission_denied_non_admin(mock_all_deps): def test_permission_denied_all_actions(mock_all_deps): mock_user.roles = [] action_endpoints = [ - ("GET", "/api/validation/tasks"), - ("POST", "/api/validation/tasks"), - ("GET", "/api/validation/tasks/task-1"), - ("PUT", "/api/validation/tasks/task-1"), - ("DELETE", "/api/validation/tasks/task-1"), - ("POST", "/api/validation/tasks/task-1/run"), - ("GET", "/api/validation/runs"), - ("GET", "/api/validation/runs/run-1"), - ("DELETE", "/api/validation/runs/run-1"), + ("GET", "/api/validation-tasks"), + ("POST", "/api/validation-tasks"), + ("GET", "/api/validation-tasks/task-1"), + ("PUT", "/api/validation-tasks/task-1"), + ("DELETE", "/api/validation-tasks/task-1"), + ("POST", "/api/validation-tasks/task-1/run"), + ("GET", "/api/validation-runs"), + ("GET", "/api/validation-runs/run-1"), + ("DELETE", "/api/validation-runs/run-1"), ] for method, path in action_endpoints: response = client.request(method, path) @@ -542,7 +544,7 @@ def test_permission_denied_all_actions(mock_all_deps): # @BRIEF delete_runs query param defaults to False. def test_delete_task_runs_default_false(mock_all_deps): mock_all_deps["task_service"].delete_task.return_value = None - response = client.delete("/api/validation/tasks/task-1") + response = client.delete("/api/validation-tasks/task-1") assert response.status_code == 204 mock_all_deps["task_service"].delete_task.assert_called_with("task-1", delete_runs=False) # #endregion test_delete_task_runs_default_false diff --git a/backend/src/api/routes/validation_tasks.py b/backend/src/api/routes/validation_tasks.py index c551db81..ce3caa95 100644 --- a/backend/src/api/routes/validation_tasks.py +++ b/backend/src/api/routes/validation_tasks.py @@ -103,6 +103,7 @@ def _dict_to_run_response(d: dict[str, Any]) -> RunResponse: # #region create_task [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Create a new validation task. # @RELATION CALLS -> [ValidationTaskService.create_task] # @RELATION CALLS -> [SchedulerService.reload_validation_policy] @@ -129,6 +130,7 @@ async def create_task( # #region list_tasks [C:3] [TYPE Function] +# @ingroup Api # @BRIEF List all validation tasks with pagination and optional filters. # @RELATION CALLS -> [ValidationTaskService.list_tasks] @router.get("", response_model=ValidationTaskListResponse) @@ -159,6 +161,7 @@ async def list_tasks( # #region get_task [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Get a validation task by policy ID. # @RELATION CALLS -> [ValidationTaskService.get_task] @router.get("/{policy_id}", response_model=ValidationTaskResponse) @@ -181,6 +184,7 @@ async def get_task( # #region update_task [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Update a validation task. # @RELATION CALLS -> [ValidationTaskService.update_task] # @RELATION CALLS -> [SchedulerService.reload_validation_policy] @@ -208,6 +212,7 @@ async def update_task( # #region delete_task [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Delete a validation task. # @RELATION CALLS -> [ValidationTaskService.delete_task] # @RELATION CALLS -> [SchedulerService.remove_validation_job] @@ -234,6 +239,7 @@ async def delete_task( # #region toggle_task_status [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Toggle a validation task's active status. # @RELATION CALLS -> [ValidationTaskService.update_task] # @RELATION CALLS -> [SchedulerService.remove_validation_job] @@ -270,6 +276,7 @@ async def toggle_task_status( # #region trigger_run [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Trigger immediate validation for a task. # @RELATION CALLS -> [ValidationTaskService.trigger_run] @router.post("/{policy_id}/run", response_model=TriggerRunResponse) @@ -300,6 +307,7 @@ async def trigger_run( # #region list_all_runs [C:3] [TYPE Function] +# @ingroup Api # @BRIEF List validation runs across all tasks with optional filters. # @RELATION CALLS -> [ValidationTaskService.list_all_runs] @router.get("/runs/all", response_model=RecordListResponse) @@ -330,6 +338,7 @@ async def list_all_runs( # #region list_runs [C:3] [TYPE Function] +# @ingroup Api # @BRIEF List run history for a validation task. # @RELATION CALLS -> [ValidationTaskService.get_run_history] @router.get("/{policy_id}/runs", response_model=RunListResponse) @@ -356,6 +365,7 @@ async def list_runs( # #region get_run_detail [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Get full run detail with all records for a validation task. # @RELATION CALLS -> [ValidationTaskService.get_run_detail] @router.get("/{policy_id}/runs/{run_id}", response_model=RunDetailResponse) @@ -388,6 +398,7 @@ async def get_run_detail( # #region parse_superset_url [C:3] [TYPE Function] +# @ingroup Api # @BRIEF Parse a Superset dashboard URL and extract dashboard context, native filters, and recovery info. # @RELATION CALLS -> [SupersetContextExtractor.parse_superset_link] @router.post("/parse-url") @@ -453,6 +464,7 @@ async def parse_superset_url( # #region legacy_llm_report_redirect [C:2] [TYPE Function] +# @ingroup Api # @BRIEF Legacy redirect: /reports/llm/{taskId} → /validation-tasks/{policy_id}/runs/{run_id} # @RELATION CALLS -> [ValidationRun] @router.get("/reports/llm/{task_id}") diff --git a/backend/src/app.py b/backend/src/app.py index b560da8e..061968e1 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,4 +1,5 @@ # #region AppModule [C:5] [TYPE Module] [SEMANTICS fastapi, websocket, startup, scheduler, middleware] +# @defgroup Module Module group. # @BRIEF The main entry point for the FastAPI application. # @LAYER API # @RELATION DEPENDS_ON -> [ApiRoutesModule] @@ -63,6 +64,7 @@ from .models.auth import Role, User # #region lifespan [C:3] [TYPE Function] +# @ingroup Module # @BRIEF Async context manager for FastAPI startup/shutdown lifecycle. # @RELATION CALLS -> [init_db] # @RELATION CALLS -> [AppDependencies] @@ -113,6 +115,7 @@ async def lifespan(app: FastAPI): scheduler.stop() # #endregion lifespan # #region FastAPI_App [C:3] [TYPE Global] [SEMANTICS app, fastapi, instance, route-registry] +# @ingroup Module # @BRIEF Canonical FastAPI application instance for route, middleware, and websocket registration. # @RELATION DEPENDS_ON -> [ApiRoutesModule] # @RELATION BINDS_TO -> [API_Routes] @@ -133,6 +136,7 @@ from .core.middleware.trace import TraceContextMiddleware # noqa: E402 app.add_middleware(TraceContextMiddleware) # #endregion FastAPI_App # #region ensure_initial_admin_user [C:3] [TYPE Function] +# @ingroup Module # @BRIEF Ensures initial admin user exists when bootstrap env flags are enabled. # @RELATION DEPENDS_ON -> [AuthRepository] def ensure_initial_admin_user() -> None: @@ -187,6 +191,7 @@ def ensure_initial_admin_user() -> None: db.close() # #endregion ensure_initial_admin_user # #region run_alembic_migrations [C:2] [TYPE Function] +# @ingroup Module # @BRIEF Applies all pending Alembic migrations against DATABASE_URL. # DEPRECATED: Migrations now run exclusively in docker/backend.entrypoint.sh. # Kept for local development (manual invocation). @@ -217,6 +222,7 @@ def run_alembic_migrations() -> None: # #region app_middleware [TYPE Block] +# @ingroup Module # @BRIEF Configure application-wide middleware (Session, CORS). # @RATIONALE SessionMiddleware uses SESSION_SECRET_KEY independent of JWT SECRET_KEY (see [SEC:H-4]). # CORS allow_origins crashes early if ALLOWED_ORIGINS is unset — no "*" fallback (see [SEC:M-1]). @@ -277,6 +283,7 @@ class HSTSMiddleware(BaseHTTPMiddleware): app.add_middleware(HSTSMiddleware) # #endregion app_middleware # #region global_exception_handler [C:2] [TYPE Function] +# @ingroup Module # @BRIEF Global exception handler — logs all unhandled 500 errors into the app logger. # @PRE request is a FastAPI Request object. # @POST Logs full traceback to superset_tools_app logger; returns 500. @@ -317,6 +324,7 @@ async def network_error_handler(request: Request, exc: NetworkError): ) # #endregion network_error_handler # #region log_requests [C:3] [TYPE Function] +# @ingroup Module # @BRIEF Middleware to log incoming HTTP requests and their response status. # @RELATION DEPENDS_ON -> [LoggerModule] # @PRE request is a FastAPI Request object. @@ -358,6 +366,7 @@ async def log_requests(request: Request, call_next): ) # #endregion log_requests # #region API_Routes [C:3] [TYPE Block] +# @ingroup Module # @BRIEF Register all FastAPI route groups exposed by the application entrypoint. # @RELATION DEPENDS_ON -> [FastAPI_App] # @RELATION DEPENDS_ON -> [Route_Group_Contracts] @@ -465,6 +474,7 @@ async def _authenticate_websocket(websocket: WebSocket, endpoint_name: str) -> b # #endregion _authenticate_websocket # #region websocket_endpoint [C:5] [TYPE Function] +# @ingroup Module # @BRIEF Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering. # @RELATION CALLS -> [TaskManagerPackage] # @RELATION DEPENDS_ON -> [LoggerModule] @@ -688,6 +698,7 @@ async def websocket_endpoint( # #endregion websocket_endpoint # #region task_events_websocket [C:4] [TYPE Function] +# @ingroup Module # @BRIEF WebSocket endpoint for global task events (status changes for ALL tasks). # @RELATION CALLS -> [TaskManagerPackage] # @PRE WebSocket must be authenticated via `token` query param. @@ -735,6 +746,7 @@ async def task_events_websocket(websocket: WebSocket): # #endregion task_events_websocket # #region maintenance_events_websocket [C:4] [TYPE Function] +# @ingroup Module # @BRIEF WebSocket endpoint for maintenance events (created/ended/banner changes). # @RELATION CALLS -> [TaskManagerPackage] # @PRE WebSocket must be authenticated via `token` query param. @@ -782,6 +794,7 @@ async def maintenance_events_websocket(websocket: WebSocket): # #endregion maintenance_events_websocket # #region dataset_websocket_endpoint [C:4] [TYPE Function] +# @ingroup Module # @BRIEF WebSocket endpoint for dataset.updated events — auto-refresh on task completion. # @RELATION CALLS -> [TaskManagerPackage] # @PRE env_id must reference a known environment. @@ -815,6 +828,7 @@ async def dataset_websocket_endpoint(websocket: WebSocket, env_id: str): logger.reflect("Released dataset event subscription", extra={"env_id": env_id}) # #endregion dataset_websocket_endpoint # #region translate_run_websocket [C:3] [TYPE Function] +# @ingroup Module # @BRIEF WebSocket endpoint for translation run progress — streams structured status updates. # @PRE run_id must be a valid translation run ID. WebSocket authenticated via `token` query param. # @POST Streams run status JSON every second until terminal state or disconnect. diff --git a/backend/src/core/__init__.py b/backend/src/core/__init__.py index 45c2cce6..cd87d2de 100644 --- a/backend/src/core/__init__.py +++ b/backend/src/core/__init__.py @@ -1,3 +1,4 @@ # #region src.core [TYPE Package] [SEMANTICS core, package, init] +# @ingroup Core # @BRIEF Backend core services and infrastructure package root. # #endregion src.core diff --git a/backend/src/core/__tests__/test_superset_preview_pipeline.py b/backend/src/core/__tests__/test_superset_preview_pipeline.py index 0f1aaf50..258c92e5 100644 --- a/backend/src/core/__tests__/test_superset_preview_pipeline.py +++ b/backend/src/core/__tests__/test_superset_preview_pipeline.py @@ -12,7 +12,7 @@ import requests from src.core.config_models import Environment from src.core.superset_client import SupersetClient from src.core.utils.async_network import AsyncAPIClient -from src.core.utils.network import APIClient, DashboardNotFoundError, SupersetAPIError +from src.core.utils.network import DashboardNotFoundError, SupersetAPIError # #region _make_environment [TYPE Function] @@ -376,41 +376,7 @@ def test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses( assert legacy_form_data["url_params"] == {"country": "DE"} assert legacy_form_data["result_type"] == "query" # #endregion test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses -# #region test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic [TYPE Function] -# @RELATION BINDS_TO -> SupersetPreviewPipelineTests -# @BRIEF Sync network client should reserve dashboard-not-found translation for dashboard endpoints only. -def test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic(): - client = APIClient( - config={ - "base_url": "http://superset.local", - "auth": {"username": "demo", "password": "secret"}, - } - ) - with pytest.raises(SupersetAPIError) as exc_info: - client._handle_http_error( - _make_requests_http_error(404, "http://superset.local/api/v1/chart/data"), - "/chart/data", - ) - assert not isinstance(exc_info.value, DashboardNotFoundError) - assert "API resource not found at endpoint '/chart/data'" in str(exc_info.value) -# #endregion test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic -# #region test_sync_network_404_mapping_translates_dashboard_endpoints [TYPE Function] -# @RELATION BINDS_TO -> SupersetPreviewPipelineTests -# @BRIEF Sync network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors. -def test_sync_network_404_mapping_translates_dashboard_endpoints(): - client = APIClient( - config={ - "base_url": "http://superset.local", - "auth": {"username": "demo", "password": "secret"}, - } - ) - with pytest.raises(DashboardNotFoundError) as exc_info: - client._handle_http_error( - _make_requests_http_error(404, "http://superset.local/api/v1/dashboard/10"), - "/dashboard/10", - ) - assert "Dashboard '/dashboard/10' Dashboard not found" in str(exc_info.value) -# #endregion test_sync_network_404_mapping_translates_dashboard_endpoints + # #region test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic [TYPE Function] # @RELATION BINDS_TO -> SupersetPreviewPipelineTests # @BRIEF Async network client should reserve dashboard-not-found translation for dashboard endpoints only. diff --git a/backend/src/core/async_superset_client.py b/backend/src/core/async_superset_client.py index d68d50ce..e770ffcb 100644 --- a/backend/src/core/async_superset_client.py +++ b/backend/src/core/async_superset_client.py @@ -1,4 +1,5 @@ # #region AsyncSupersetClientModule [C:3] [TYPE Module] [SEMANTICS superset, transform, dashboard, filter, async-superset-client] +# @defgroup Core Module group. # @BRIEF Backward-compatible alias for the async-migrated SupersetClient. # @DEPRECATED 2026-06-04 — class kept as thin compat wrapper. New code should import SupersetClient directly. # @REPLACED_BY -> [SupersetClient] @@ -13,6 +14,7 @@ from .utils.async_network import AsyncAPIClient # #region AsyncSupersetClient [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Backward-compatible alias for the async-migrated SupersetClient. # @RELATION INHERITS -> [SupersetClient] # @RELATION DEPENDS_ON -> [AsyncAPIClient] @@ -25,6 +27,7 @@ class AsyncSupersetClient(SupersetClient): """ # #region AsyncSupersetClientInit [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Initialize async Superset client — delegates to SupersetClient. # @PRE env is valid Environment instance. # @POST Client uses AsyncAPIClient transport. @@ -35,6 +38,7 @@ class AsyncSupersetClient(SupersetClient): # #endregion AsyncSupersetClientInit # #region AsyncSupersetClientClose [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Close async transport resources. # @POST Underlying AsyncAPIClient is closed. # @SIDE_EFFECT Closes network sockets. diff --git a/backend/src/core/auth/__init__.py b/backend/src/core/auth/__init__.py index 673e3a16..26a9a2ed 100644 --- a/backend/src/core/auth/__init__.py +++ b/backend/src/core/auth/__init__.py @@ -1,3 +1,4 @@ # #region AuthPackage [TYPE Package] [SEMANTICS auth, package, init] +# @ingroup Auth # @BRIEF Authentication and authorization package root. # #endregion AuthPackage diff --git a/backend/src/core/auth/api_key.py b/backend/src/core/auth/api_key.py index e72490e8..e2771ff3 100644 --- a/backend/src/core/auth/api_key.py +++ b/backend/src/core/auth/api_key.py @@ -1,4 +1,5 @@ # #region APIKeyUtilities [C:2] [TYPE Module] [SEMANTICS auth, api_key, crypto, generation] +# @defgroup Auth Module group. # @BRIEF API key generation and hashing utilities for service-to-service authentication. # @LAYER Core # @RELATION DEPENDS_ON -> [EXT:Python:hashlib] @@ -11,6 +12,7 @@ import secrets # #region generate_api_key [C:2] [TYPE Function] +# @ingroup Auth # @BRIEF Generate a new API key in ssk_ format with SHA-256 hash. # @POST Returns (raw_key, prefix, key_hash) — raw_key shown ONCE to caller, never stored. # @SIDE_EFFECT Uses secrets.token_urlsafe(32) for cryptographic randomness. diff --git a/backend/src/core/auth/config.py b/backend/src/core/auth/config.py index 39c67dfc..0cd1d438 100644 --- a/backend/src/core/auth/config.py +++ b/backend/src/core/auth/config.py @@ -1,4 +1,5 @@ # #region AuthConfigModule [C:2] [TYPE Module] [SEMANTICS pydantic, auth, auth-config, config] +# @defgroup Auth Module group. # # @BRIEF Centralized configuration for authentication and authorization. # @LAYER Core @@ -17,6 +18,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict # #region AuthConfig [TYPE Class] +# @defgroup Auth Module group. # @BRIEF Holds authentication-related settings. # @PRE Environment variables may be provided via .env file. # @POST Returns a configuration object with validated settings. @@ -65,6 +67,7 @@ class AuthConfig(BaseSettings): # #endregion AuthConfig # #region auth_config [TYPE Variable] +# @ingroup Auth # @BRIEF Singleton instance of AuthConfig. # @RELATION DEPENDS_ON -> AuthConfig auth_config = AuthConfig() diff --git a/backend/src/core/auth/jwt.py b/backend/src/core/auth/jwt.py index 013e7a9c..15c0dce8 100644 --- a/backend/src/core/auth/jwt.py +++ b/backend/src/core/auth/jwt.py @@ -1,16 +1,15 @@ -# #region AuthJwtModule [C:5] [TYPE Module] [SEMANTICS auth, validate, jwt, token] -# -# @BRIEF JWT token generation and validation logic with server-side revocation. +# #region Auth.Jwt [C:5] [TYPE Module] [SEMANTICS auth,jwt,token,security] +# @defgroup Auth JWT token generation, validation, and revocation. # @LAYER Core -# @RELATION DEPENDS_ON -> [auth_config] -# @RELATION DEPENDS_ON -> [TokenBlacklist] -# -# @INVARIANT Tokens must include aud, iss, iat, jti, exp, and sub claims. -# Revoked tokens are rejected even if not expired. -# @PRE JWT secret configured in environment -# @POST Token encode/decode/revoke functions exported -# @SIDE_EFFECT None -# @DATA_CONTRACT TokenPayload -> JWT string +# @RELATION DEPENDS_ON -> [Auth.Config] +# @RELATION DEPENDS_ON -> [Auth.TokenBlacklist] +# @INVARIANT Tokens must include aud, iss, iat, jti, exp, and sub claims. +# @INVARIANT Revoked tokens are rejected even if not expired. +# @PRE JWT secret configured in environment. +# @POST Token encode/decode/revoke functions exported. +# @DATA_CONTRACT TokenPayload -> JWT string +# @RATIONALE JWT chosen over opaque tokens for stateless validation — eliminates DB lookup on every request. Server-side blacklist added for immediate revocation (logout, password change). +# @REJECTED Session-based auth rejected — requires server-side state per request. Opaque tokens rejected — every validation requires DB round-trip. from datetime import UTC, datetime, timedelta import hashlib @@ -24,11 +23,13 @@ from ..logger import belief_scope from .config import auth_config -# #region create_access_token [TYPE Function] +# #region Auth.Jwt.CreateAccessToken [C:4] [TYPE Function] [SEMANTICS auth,jwt,token] +# @ingroup Auth # @BRIEF Generates a new JWT access token with aud, iss, iat, jti, exp claims. -# @PRE data dict contains 'sub' (user_id) and optional 'scopes' (roles). -# @POST Returns a signed JWT string with unique jti for revocation. -# @RELATION DEPENDS_ON -> [auth_config] +# @PRE data dict contains 'sub' (user_id) and optional 'scopes'. +# @POST Returns a signed JWT string with unique jti for revocation. +# @SIDE_EFFECT None (pure function). +# @RELATION DEPENDS_ON -> [Auth.Config] # def create_access_token(data: dict, expires_delta: timedelta | None = None) -> str: with belief_scope("create_access_token"): @@ -54,14 +55,16 @@ def create_access_token(data: dict, expires_delta: timedelta | None = None) -> s return encoded_jwt -# #endregion create_access_token +# #endregion Auth.Jwt.CreateAccessToken -# #region decode_token [TYPE Function] +# #region Auth.Jwt.DecodeToken [C:4] [TYPE Function] [SEMANTICS auth,jwt,validate] +# @ingroup Auth # @BRIEF Decodes and validates a JWT token — verifies aud, iss, exp, and blacklist. -# @PRE token is a signed JWT string. -# @POST Returns the decoded payload if valid. Raises JWTError if invalid. -# @RELATION DEPENDS_ON -> [auth_config] +# @PRE token is a signed JWT string. +# @POST Returns decoded payload if valid. Raises JWTError if invalid/revoked. +# @RELATION DEPENDS_ON -> [Auth.Config] +# @RELATION CALLS -> [Auth.Jwt.IsTokenBlacklisted] # def decode_token(token: str) -> dict: with belief_scope("decode_token"): @@ -81,39 +84,41 @@ def decode_token(token: str) -> dict: return payload -# #endregion decode_token +# #endregion Auth.Jwt.DecodeToken -# #region _hash_token [TYPE Function] +# #region Auth.Jwt._HashToken [C:2] [TYPE Function] # @BRIEF SHA-256 hash of a token string for blacklist lookup. -# @PRE token is a non-empty JWT string. -# @POST Returns 64-character hex digest. +# @PRE token is a non-empty JWT string. +# @POST Returns 64-character hex digest. def _hash_token(token: str) -> str: return hashlib.sha256(token.encode()).hexdigest() - -# #endregion _hash_token +# #endregion Auth.Jwt._HashToken -# #region _prune_blacklist [TYPE Function] +# #region Auth.Jwt._PruneBlacklist [C:2] [TYPE Function] # @BRIEF Delete expired entries from the token blacklist. -# @PRE db is a valid SQLAlchemy session. -# @POST Expired blacklist rows are removed. +# @PRE db is a valid SQLAlchemy session. +# @POST Expired blacklist rows are removed. +# @SIDE_EFFECT Mutates token_blacklist table. def _prune_blacklist(db: Session) -> None: now = datetime.now(UTC) db.query(TokenBlacklist).filter(TokenBlacklist.expires_at < now).delete() db.commit() - -# #endregion _prune_blacklist +# #endregion Auth.Jwt._PruneBlacklist -# #region blacklist_token [TYPE Function] +# #region Auth.Jwt.BlacklistToken [C:4] [TYPE Function] [SEMANTICS auth,jwt,revoke] +# @ingroup Auth # @BRIEF Revoke a JWT token by adding its hash to the blacklist. -# @PRE token was issued by this service and hasn't expired. -# @POST Token is blacklisted and subsequent decode_token calls will reject it. -# @SIDE_EFFECT Writes to token_blacklist table; prunes expired entries. -# @RELATION DEPENDS_ON -> [TokenBlacklist] +# @PRE token was issued by this service and hasn't expired. +# @POST Token is blacklisted; subsequent decode attempts will reject it. +# @SIDE_EFFECT Writes to token_blacklist table; prunes expired entries. +# @RELATION DEPENDS_ON -> [Auth.TokenBlacklist] +# @RELATION CALLS -> [Auth.Jwt.HashToken] +# @TEST_EDGE: already_expired -> skips blacklisting def blacklist_token(token: str, db: Session) -> None: with belief_scope("blacklist_token"): _prune_blacklist(db) @@ -145,14 +150,15 @@ def blacklist_token(token: str, db: Session) -> None: db.commit() -# #endregion blacklist_token +# #endregion Auth.Jwt.BlacklistToken -# #region is_token_blacklisted [TYPE Function] -# @BRIEF Check if a JWT token has been revoked. -# @PRE db is a valid SQLAlchemy session. -# @POST Returns True if token is blacklisted, False otherwise. -# @RELATION DEPENDS_ON -> [TokenBlacklist] +# #region Auth.Jwt.IsTokenBlacklisted [C:2] [TYPE Function] [SEMANTICS auth,jwt,revoke] +# @ingroup Auth +# @BRIEF Check if a JWT token hash exists in the blacklist. +# @PRE db is a valid SQLAlchemy session. +# @POST Returns True if token is blacklisted, False otherwise. +# @RELATION DEPENDS_ON -> [Auth.TokenBlacklist] def is_token_blacklisted(token: str, db: Session) -> bool: with belief_scope("is_token_blacklisted"): try: @@ -167,6 +173,6 @@ def is_token_blacklisted(token: str, db: Session) -> bool: return db.query(TokenBlacklist).filter(TokenBlacklist.jti == jti).first() is not None -# #endregion is_token_blacklisted +# #endregion Auth.Jwt.IsTokenBlacklisted -# #endregion AuthJwtModule +# #endregion Auth.Jwt diff --git a/backend/src/core/auth/logger.py b/backend/src/core/auth/logger.py index a28407eb..b1f82b62 100644 --- a/backend/src/core/auth/logger.py +++ b/backend/src/core/auth/logger.py @@ -1,4 +1,5 @@ # #region AuthLoggerModule [C:5] [TYPE Module] [SEMANTICS auth, audit, logging] +# @defgroup Auth Module group. # # @BRIEF Structured auth logging module for audit trail generation. # @LAYER Core @@ -41,6 +42,7 @@ def _mask_details(details: dict[str, Any] | None) -> dict[str, Any] | None: # #region log_security_event [TYPE Function] +# @ingroup Auth # @BRIEF Logs a security-related event for audit trails. # @PRE event_type and username are strings. # @POST Security event is written to the application log via %s-formatting to prevent log injection. diff --git a/backend/src/core/auth/oauth.py b/backend/src/core/auth/oauth.py index d19f6c54..9a4fc245 100644 --- a/backend/src/core/auth/oauth.py +++ b/backend/src/core/auth/oauth.py @@ -1,4 +1,5 @@ # #region AuthOauthModule [C:2] [TYPE Module] [SEMANTICS auth, oauth, oidc, adfs, authlib] +# @defgroup Auth Module group. # # @BRIEF ADFS OIDC configuration and client using Authlib. # @LAYER Core @@ -12,12 +13,14 @@ from authlib.integrations.starlette_client import OAuth from .config import auth_config # #region oauth [TYPE Variable] +# @ingroup Auth # @BRIEF Global Authlib OAuth registry. # @RELATION DEPENDS_ON -> [EXT:Library:OAuth] oauth = OAuth() # #endregion oauth # #region register_adfs [TYPE Function] +# @ingroup Auth # @BRIEF Registers the ADFS OIDC client. # @PRE ADFS configuration is provided in auth_config. # @POST ADFS client is registered in oauth registry. @@ -37,6 +40,7 @@ def register_adfs(): # #endregion register_adfs # #region is_adfs_configured [TYPE Function] +# @ingroup Auth # @BRIEF Checks if ADFS is properly configured. # @PRE None. # @POST Returns True if ADFS client is registered, False otherwise. diff --git a/backend/src/core/auth/repository.py b/backend/src/core/auth/repository.py index 51083b72..a7e0b78a 100644 --- a/backend/src/core/auth/repository.py +++ b/backend/src/core/auth/repository.py @@ -1,4 +1,5 @@ # #region AuthRepositoryModule [C:5] [TYPE Module] [SEMANTICS sqlalchemy, auth, search, user, auth-repository] +# @defgroup Auth Module group. # @BRIEF Data access layer for authentication and user preference entities. # @LAYER Domain # @RELATION DEPENDS_ON -> [AuthModels] @@ -18,6 +19,7 @@ from ..logger import belief_scope, logger # #region AuthRepository [TYPE Class] +# @defgroup Auth Module group. # @BRIEF Provides low-level CRUD operations for identity and authorization records. # @PRE Database session is bound. # @POST Entity instances returned safely. @@ -27,6 +29,7 @@ class AuthRepository: def __init__(self, db: Session): self.db = db # #region get_user_by_id [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve user by UUID. # @PRE user_id is a valid UUID string. # @POST Returns User object if found, else None. @@ -39,6 +42,7 @@ class AuthRepository: return result # #endregion get_user_by_id # #region get_user_by_username [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve user by username. # @PRE username is a non-empty string. # @POST Returns User object if found, else None. @@ -51,6 +55,7 @@ class AuthRepository: return result # #endregion get_user_by_username # #region get_role_by_id [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve role by UUID with permissions preloaded. # @RELATION DEPENDS_ON -> [Role] # @RELATION DEPENDS_ON -> [Permission] @@ -64,6 +69,7 @@ class AuthRepository: ) # #endregion get_role_by_id # #region get_role_by_name [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve role by unique name. # @RELATION DEPENDS_ON -> [Role] def get_role_by_name(self, name: str) -> Role | None: @@ -71,6 +77,7 @@ class AuthRepository: return self.db.query(Role).filter(Role.name == name).first() # #endregion get_role_by_name # #region get_permission_by_id [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve permission by UUID. # @RELATION DEPENDS_ON -> [Permission] def get_permission_by_id(self, permission_id: str) -> Permission | None: @@ -80,6 +87,7 @@ class AuthRepository: ) # #endregion get_permission_by_id # #region get_permission_by_resource_action [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve permission by resource and action tuple. # @RELATION DEPENDS_ON -> [Permission] def get_permission_by_resource_action( @@ -93,6 +101,7 @@ class AuthRepository: ) # #endregion get_permission_by_resource_action # #region list_permissions [TYPE Function] +# @ingroup Auth # @PURPOSE: List all system permissions. # @RELATION DEPENDS_ON -> [Permission] def list_permissions(self) -> list[Permission]: @@ -100,6 +109,7 @@ class AuthRepository: return self.db.query(Permission).all() # #endregion list_permissions # #region get_user_dashboard_preference [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve dashboard filters/preferences for a user. # @RELATION DEPENDS_ON -> [UserDashboardPreference] def get_user_dashboard_preference( @@ -113,6 +123,7 @@ class AuthRepository: ) # #endregion get_user_dashboard_preference # #region get_roles_by_ad_groups [TYPE Function] +# @ingroup Auth # @PURPOSE: Retrieve roles that match a list of AD group names. # @PRE groups is a list of strings representing AD group identifiers. # @POST Returns a list of Role objects mapped to the provided AD groups. diff --git a/backend/src/core/auth/security.py b/backend/src/core/auth/security.py index d1790332..149dcc2a 100644 --- a/backend/src/core/auth/security.py +++ b/backend/src/core/auth/security.py @@ -1,4 +1,5 @@ # #region AuthSecurityModule [C:2] [TYPE Module] [SEMANTICS auth, password, hashing, bcrypt, security] +# @defgroup Auth Module group. # # @BRIEF Utility for password hashing and verification using Passlib. # @LAYER Core @@ -10,6 +11,7 @@ import bcrypt # #region verify_password [TYPE Function] +# @ingroup Auth # @BRIEF Verifies a plain password against a hashed password. # @PRE plain_password is a string, hashed_password is a bcrypt hash. # @POST Returns True if password matches, False otherwise. @@ -28,6 +30,7 @@ def verify_password(plain_password: str, hashed_password: str) -> bool: # #endregion verify_password # #region get_password_hash [TYPE Function] +# @ingroup Auth # @BRIEF Generates a bcrypt hash for a plain password. # @PRE password is a string. # @POST Returns a secure bcrypt hash string. diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py index 5325e6e9..9a885137 100644 --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -1,4 +1,5 @@ # #region ConfigManager [C:5] [TYPE Module] [SEMANTICS sqlalchemy, validate, migration, config-manager] +# @defgroup Core Module group. # # @BRIEF Manages application configuration persistence in DB with one-time migration from legacy JSON. # @LAYER Domain @@ -31,6 +32,7 @@ from .logger import belief_scope, configure_logger, logger # #region ConfigManager [C:5] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Handles application configuration load, validation, mutation, and persistence lifecycle. # @PRE Database is accessible and AppConfigRecord schema is loaded. # @POST Configuration state is synchronized between memory and database. @@ -446,6 +448,7 @@ class ConfigManager: db.close() # #endregion _save_config_to_db # #region save [TYPE Function] +# @ingroup Core # @PURPOSE: Persist current in-memory configuration state. def save(self) -> None: with belief_scope("ConfigManager.save"): @@ -453,18 +456,21 @@ class ConfigManager: self._save_config_to_db(self.config) # #endregion save # #region get_config [TYPE Function] +# @ingroup Core # @PURPOSE: Return current in-memory configuration snapshot. def get_config(self) -> AppConfig: with belief_scope("ConfigManager.get_config"): return self.config # #endregion get_config # #region get_payload [TYPE Function] +# @ingroup Core # @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema. def get_payload(self) -> dict[str, Any]: with belief_scope("ConfigManager.get_payload"): return self._sync_raw_payload_from_config() # #endregion get_payload # #region save_config [TYPE Function] +# @ingroup Core # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict. def save_config(self, config: Any) -> AppConfig: with belief_scope("ConfigManager.save_config"): @@ -496,6 +502,7 @@ class ConfigManager: raise TypeError("config must be AppConfig or dict") # #endregion save_config # #region update_global_settings [TYPE Function] +# @ingroup Core # @PURPOSE: Replace global settings and persist the resulting configuration. def update_global_settings(self, settings: GlobalSettings) -> AppConfig: with belief_scope("ConfigManager.update_global_settings"): @@ -505,6 +512,7 @@ class ConfigManager: return self.config # #endregion update_global_settings # #region validate_path [TYPE Function] +# @ingroup Core # @PURPOSE: Validate that path exists and is writable, creating it when absent. def validate_path(self, path: str) -> tuple[bool, str]: with belief_scope("ConfigManager.validate_path", f"path={path}"): @@ -528,18 +536,21 @@ class ConfigManager: return False, str(exc) # #endregion validate_path # #region get_environments [TYPE Function] +# @ingroup Core # @PURPOSE: Return all configured environments. def get_environments(self) -> list[Environment]: with belief_scope("ConfigManager.get_environments"): return list(self.config.environments) # #endregion get_environments # #region has_environments [TYPE Function] +# @ingroup Core # @PURPOSE: Check whether at least one environment exists in configuration. def has_environments(self) -> bool: with belief_scope("ConfigManager.has_environments"): return len(self.config.environments) > 0 # #endregion has_environments # #region get_environment [TYPE Function] +# @ingroup Core # @PURPOSE: Resolve a configured environment by identifier. def get_environment(self, env_id: str) -> Environment | None: with belief_scope("ConfigManager.get_environment", f"env_id={env_id}"): @@ -552,6 +563,7 @@ class ConfigManager: return None # #endregion get_environment # #region add_environment [TYPE Function] +# @ingroup Core # @PURPOSE: Upsert environment by id into configuration and persist. def add_environment(self, env: Environment) -> AppConfig: with belief_scope("ConfigManager.add_environment", f"env_id={env.id}"): @@ -583,6 +595,7 @@ class ConfigManager: return self.config # #endregion add_environment # #region update_environment [TYPE Function] +# @ingroup Core # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior. def update_environment(self, env_id: str, env: Environment) -> bool: with belief_scope("ConfigManager.update_environment", f"env_id={env_id}"): @@ -608,6 +621,7 @@ class ConfigManager: return False # #endregion update_environment # #region delete_environment [TYPE Function] +# @ingroup Core # @PURPOSE: Delete environment by id and persist when deletion occurs. def delete_environment(self, env_id: str) -> bool: with belief_scope("ConfigManager.delete_environment", f"env_id={env_id}"): diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 6b08632e..f06d0a01 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -1,14 +1,17 @@ # #region ConfigModels [C:3] [TYPE Module] [SEMANTICS pydantic, model, schedule] +# @defgroup Core Module group. # @BRIEF Defines the data models for application configuration using Pydantic. # @LAYER Core # @RELATION IMPLEMENTS -> [CoreContracts] # @RELATION IMPLEMENTS -> [ConnectionContracts] # # #region CoreContracts [C:2] [TYPE Interface] +# @ingroup Core # @BRIEF Contract for core configuration models — Pydantic BaseModel subclasses with scheduler config. # #endregion CoreContracts # # #region ConnectionContracts [C:2] [TYPE Interface] +# @ingroup Core # @BRIEF Contract for database/environment connection configuration models. # #endregion ConnectionContracts @@ -23,6 +26,7 @@ from ..services.llm_prompt_templates import ( # #region Schedule [TYPE DataClass] +# @ingroup Core # @BRIEF Represents a backup schedule configuration. class Schedule(BaseModel): enabled: bool = False @@ -33,6 +37,7 @@ class Schedule(BaseModel): # #region Environment [TYPE DataClass] +# @ingroup Core # @BRIEF Represents a Superset environment configuration with async connection pool settings. class Environment(BaseModel): id: str @@ -70,6 +75,7 @@ class Environment(BaseModel): # #region AppAsyncRuntimeConfig [TYPE DataClass] +# @ingroup Core # @BRIEF Global application-level async runtime configuration. # @RATIONALE Separated from EnvironmentConfig because these settings are not per-env: # executor workers, shutdown timeout, blocking queue timeout are app-wide. @@ -104,6 +110,7 @@ class AppAsyncRuntimeConfig(BaseModel): # #region LoggingConfig [TYPE DataClass] +# @ingroup Core # @BRIEF Defines the configuration for the application's logging system. class LoggingConfig(BaseModel): level: str = "INFO" @@ -120,6 +127,7 @@ class LoggingConfig(BaseModel): # #region CleanReleaseConfig [TYPE DataClass] +# @ingroup Core # @BRIEF Configuration for clean release compliance subsystem. class CleanReleaseConfig(BaseModel): active_policy_id: str | None = None @@ -142,6 +150,7 @@ class FeaturesConfig(BaseModel): # #region GlobalSettings [TYPE DataClass] +# @ingroup Core # @BRIEF Represents global application settings. class GlobalSettings(BaseModel): storage: StorageConfig = Field(default_factory=StorageConfig) @@ -218,6 +227,7 @@ class GlobalSettings(BaseModel): # #region AppConfig [TYPE DataClass] +# @ingroup Core # @BRIEF The root configuration model containing all application settings. class AppConfig(BaseModel): environments: list[Environment] = [] diff --git a/backend/src/core/cot_logger.py b/backend/src/core/cot_logger.py index 105442ab..b089c09b 100644 --- a/backend/src/core/cot_logger.py +++ b/backend/src/core/cot_logger.py @@ -1,4 +1,5 @@ # #region CotLoggerModule [C:4] [TYPE Module] [SEMANTICS logging, cot, json, trace, structured] +# @defgroup Core Module group. # @BRIEF Structured JSON logger implementing the decision audit logging protocol. # Uses ContextVar for trace_id and span_id propagation across async contexts. # Provides log(), MarkerLogger, seed_trace_id(), push_span(), pop_span(). @@ -112,6 +113,7 @@ def pop_span(prev: str) -> None: # #endregion pop_span # #region cot_log_function [C:2] [TYPE Function] [SEMANTICS log, json, structured, marker] +# @ingroup Core # @BRIEF Core structured logging function that emits a single-line JSON record. def log( src: str, @@ -165,6 +167,7 @@ def log( # #endregion cot_log_function # #region MarkerLogger [C:2] [TYPE Class] [SEMANTICS logger, proxy, marker, syntactic-sugar] +# @defgroup Core Module group. # @BRIEF Thin proxy class providing .reason(), .reflect(), .explore() convenience methods. class MarkerLogger: """Thin proxy over the cot_logger.log() function. @@ -180,6 +183,7 @@ class MarkerLogger: """ # #region MarkerLogger.__init__ [TYPE Function] +# @ingroup Core # @BRIEF Store the module/component name used as the 'src' field in all log calls. def __init__(self, module_name: str) -> None: """Initialise the MarkerLogger with a module or component name. @@ -192,6 +196,7 @@ class MarkerLogger: # #endregion MarkerLogger.__init__ # #region MarkerLogger.reason [TYPE Function] +# @ingroup Core # @BRIEF Log a REASON marker — strict deduction, core logic. def reason( self, intent: str, *, payload: dict[str, Any] | None = None @@ -202,6 +207,7 @@ class MarkerLogger: # #endregion MarkerLogger.reason # #region MarkerLogger.reflect [TYPE Function] +# @ingroup Core # @BRIEF Log a REFLECT marker — self-check, structural validation. def reflect( self, intent: str, *, payload: dict[str, Any] | None = None @@ -212,6 +218,7 @@ class MarkerLogger: # #endregion MarkerLogger.reflect # #region MarkerLogger.explore [TYPE Function] +# @ingroup Core # @BRIEF Log an EXPLORE marker — searching, alternatives, violated assumptions. def explore( self, diff --git a/backend/src/core/database.py b/backend/src/core/database.py index 0beadcbd..5c95ddba 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -1,4 +1,5 @@ # #region DatabaseModule [C:3] [TYPE Module] [SEMANTICS sqlalchemy, connection, session] +# @defgroup Core Module group. # # @BRIEF Configures database connection and session management (PostgreSQL-first). # @LAYER Infrastructure @@ -760,6 +761,7 @@ def _ensure_dictionary_entries_columns(bind_engine): # #region init_db [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Creates any missing tables via create_all() — safety net for development. # @PRE engine, tasks_engine and auth_engine are initialized. # In Docker: Alembic migrations already applied via entrypoint.sh. @@ -785,6 +787,7 @@ def init_db(): # #region get_db [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Dependency for getting a database session. # @PRE SessionLocal is initialized. # @POST Session is closed after use. @@ -802,6 +805,7 @@ def get_db(): # #region get_tasks_db [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Dependency for getting a tasks database session. # @PRE TasksSessionLocal is initialized. # @POST Session is closed after use. @@ -819,6 +823,7 @@ def get_tasks_db(): # #region get_auth_db [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Dependency for getting an authentication database session. # @PRE AuthSessionLocal is initialized. # @POST Session is closed after use. diff --git a/backend/src/core/encryption.py b/backend/src/core/encryption.py index e36830c4..13cd1190 100644 --- a/backend/src/core/encryption.py +++ b/backend/src/core/encryption.py @@ -1,4 +1,5 @@ # #region EncryptionCore [C:5] [TYPE Module] [SEMANTICS encryption, fernet, key, crypto, secret] +# @defgroup Core Module group. # @BRIEF Core encryption infrastructure: Fernet-based EncryptionManager for reversible secret storage. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [LoggerModule] @@ -55,6 +56,7 @@ def _require_fernet_key() -> bytes: # #region EncryptionManager [C:5] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Handles encryption and decryption of sensitive data like API keys and passwords. # @RELATION CALLS -> [_require_fernet_key] # @PRE ENCRYPTION_KEY is configured with a valid Fernet key before instantiation. diff --git a/backend/src/core/encryption_key.py b/backend/src/core/encryption_key.py index 32488b90..3668d16f 100644 --- a/backend/src/core/encryption_key.py +++ b/backend/src/core/encryption_key.py @@ -1,4 +1,5 @@ # #region EncryptionKeyModule [C:5] [TYPE Module] [SEMANTICS encryption, fernet, key, env, secret] +# @defgroup Core Module group. # @BRIEF Resolve a Fernet encryption key from environment or .env file. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [LoggerModule] @@ -27,6 +28,7 @@ DEFAULT_ENV_FILE_PATH = Path(__file__).resolve().parents[2] / ".env" # #region ensure_encryption_key [TYPE Function] +# @ingroup Core # @BRIEF Ensure backend runtime has a persistent valid Fernet key. # @PRE ENCRYPTION_KEY is set in process environment or backend/.env file. # @POST Returns a valid Fernet key and sets it in process environment. diff --git a/backend/src/core/logger.py b/backend/src/core/logger.py index a9e06688..893bf857 100755 --- a/backend/src/core/logger.py +++ b/backend/src/core/logger.py @@ -1,4 +1,5 @@ # #region LoggerModule [C:5] [TYPE Module] [SEMANTICS pydantic, logging, json, formatter, structured] +# @defgroup Core Module group. # @BRIEF Application logging system with CotJsonFormatter producing decision audit JSON output. # @LAYER Core # @RELATION CALLED_BY -> [LoggerModule] @@ -47,6 +48,7 @@ _task_log_level = "INFO" # #region CotJsonFormatter [C:3] [TYPE Class] [SEMANTICS logging,formatter,json,cot,protocol] +# @defgroup Core Module group. # @BRIEF JSON formatter implementing the decision audit logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id. # @RELATION IMPLEMENTS -> [CotJsonFormatter] class CotJsonFormatter(logging.Formatter): @@ -72,6 +74,7 @@ class CotJsonFormatter(logging.Formatter): """ # #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record] +# @ingroup Core # @BRIEF Format a LogRecord into a single-line CoT JSON string with decision audit protocol fields. # @INVARIANT Output is always valid single-line JSON. def format(self, record): @@ -111,6 +114,7 @@ class CotJsonFormatter(logging.Formatter): # #region BeliefFormatter [C:2] [TYPE Class] [SEMANTICS logging,formatter,legacy,text] +# @defgroup Core Module group. # @BRIEF Legacy formatter that adds belief state prefixes to log messages. Deprecated in favor of CotJsonFormatter. # @RATIONALE Kept for backward compatibility; may be used by external consumers of this module. class BeliefFormatter(logging.Formatter): @@ -143,6 +147,7 @@ class LogEntry(BaseModel): # #region belief_scope [C:3] [TYPE Function] [SEMANTICS logging,context,belief_state,cot] +# @ingroup Core # @BRIEF Context manager for decision audit structured logging. Uses cot_logger.log() for JSON output. # @PRE anchor_id must be provided. # @POST Thread-local belief state is updated; REASON/REFLECT/EXPLORE markers emitted via structured JSON logger. @@ -169,6 +174,7 @@ def belief_scope(anchor_id: str, message: str = ""): # #region configure_logger [C:4] [TYPE Function] [SEMANTICS logging,configuration,initialization,cot] +# @ingroup Core # @BRIEF Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags. # @RELATION CALLS -> [CotJsonFormatter] # @RELATION CALLS -> [CotLoggerModule] @@ -253,10 +259,12 @@ def should_log_task_level(level: str) -> bool: # #region Logger [C:2] [TYPE Global] [SEMANTICS logger,global,instance] +# @ingroup Core # @BRIEF The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler. logger = logging.getLogger("superset_tools_app") # #region believed [C:2] [TYPE Function] [SEMANTICS decorator,belief,scope] +# @ingroup Core # @BRIEF A decorator that wraps a function in a belief scope. def believed(anchor_id: str): def decorator(func): @@ -287,6 +295,7 @@ logger.addHandler(websocket_log_handler) # #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning] +# @ingroup Core # @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter. def explore(self, msg, *args, **kwargs): """Log an EXPLORE marker — searching, alternatives, violated assumptions. @@ -304,6 +313,7 @@ def explore(self, msg, *args, **kwargs): # #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info] +# @ingroup Core # @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter. # @RATIONALE INFO level per decision audit protocol: REASON is the primary reasoning bond # and must be visible in production. DEBUG is reserved for high-frequency loops. @@ -323,6 +333,7 @@ def reason(self, msg, *args, **kwargs): # #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,info] +# @ingroup Core # @BRIEF Logs a REFLECT marker (INFO level) with structured extra data for CotJsonFormatter. # @RATIONALE INFO level per decision audit protocol: REFLECT verifies outcomes and # must be visible in production. DEBUG is reserved for high-frequency loops. diff --git a/backend/src/core/mapping_service.py b/backend/src/core/mapping_service.py index e48e02e9..45412bd9 100644 --- a/backend/src/core/mapping_service.py +++ b/backend/src/core/mapping_service.py @@ -1,4 +1,5 @@ # #region IdMappingServiceModule [C:5] [TYPE Module] [SEMANTICS sqlalchemy, mapping, sync, schedule, superset, resource] +# @defgroup Core Module group. # # @BRIEF Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID) # @LAYER Core @@ -23,6 +24,7 @@ from src.models.mapping import ResourceMapping, ResourceType # #region IdMappingService [C:5] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs. # @PRE db_session is an active SQLAlchemy Session bound to mapping tables. # @POST Service instance provides scheduler control and environment-scoped mapping synchronization APIs. @@ -55,6 +57,7 @@ class IdMappingService: self._sync_job = None # #endregion __init__ # #region start_scheduler [TYPE Function] +# @ingroup Core # @PURPOSE: Starts the background scheduler with a given cron string. # @PARAM cron_string (str) - Cron expression for the sync interval. # @PARAM environments (List[str]) - List of environment IDs to sync. @@ -92,6 +95,7 @@ class IdMappingService: ) # #endregion start_scheduler # #region sync_environment [TYPE Function] +# @ingroup Core # @PURPOSE: Fully synchronizes mapping for a specific environment. # @PARAM environment_id (str) - Target environment ID. # @PARAM superset_client - Instance capable of hitting the Superset API. @@ -225,6 +229,7 @@ class IdMappingService: raise # #endregion sync_environment # #region get_remote_id [TYPE Function] +# @ingroup Core # @PURPOSE: Retrieves the remote integer ID for a given universal UUID. # @PARAM environment_id (str) # @PARAM resource_type (ResourceType) @@ -248,6 +253,7 @@ class IdMappingService: return None # #endregion get_remote_id # #region get_remote_ids_batch [TYPE Function] +# @ingroup Core # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently. # @PARAM environment_id (str) # @PARAM resource_type (ResourceType) diff --git a/backend/src/core/middleware/trace.py b/backend/src/core/middleware/trace.py index 75e350a8..cdc111a5 100644 --- a/backend/src/core/middleware/trace.py +++ b/backend/src/core/middleware/trace.py @@ -1,4 +1,5 @@ # #region TraceContextMiddlewareModule [C:3] [TYPE Module] [SEMANTICS fastapi, middleware, trace, context, request] +# @defgroup Core Module group. # @BRIEF Raw ASGI middleware that seeds a trace_id for every incoming HTTP request. # Optionally extracts X-Trace-ID header for cross-service trace propagation. # Implemented as raw ASGI middleware (not BaseHTTPMiddleware) to avoid @@ -41,11 +42,13 @@ from ..cot_logger import seed_trace_id, set_trace_id # #region TraceContextMiddleware [C:2] [TYPE Class] [SEMANTICS middleware, trace, context, asgi] +# @defgroup Core Module group. # @BRIEF Raw ASGI middleware that seeds a trace_id per request. Runs in the root task # context, ensuring all downstream BaseHTTPMiddleware layers see the trace_id. class TraceContextMiddleware: # #region TraceContextMiddleware.__init__ [TYPE Function] +# @ingroup Core # @BRIEF Store the inner ASGI app. def __init__(self, app: ASGIApp) -> None: self.app = app @@ -53,6 +56,7 @@ class TraceContextMiddleware: # #endregion TraceContextMiddleware.__init__ # #region TraceContextMiddleware.__call__ [C:2] [TYPE Function] [SEMANTICS asgi, call, trace, seed, header] +# @ingroup Core # @BRIEF ASGI __call__ that seeds trace_id in the root task context before forwarding. async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] != "http": diff --git a/backend/src/core/migration/__init__.py b/backend/src/core/migration/__init__.py index 52848a2a..78da7c67 100644 --- a/backend/src/core/migration/__init__.py +++ b/backend/src/core/migration/__init__.py @@ -1,4 +1,5 @@ # #region MigrationPackage [TYPE Module] [SEMANTICS migration, package, init] +# @defgroup Migration Module group. from .archive_parser import MigrationArchiveParser from .dry_run_orchestrator import MigrationDryRunService diff --git a/backend/src/core/migration/archive_parser.py b/backend/src/core/migration/archive_parser.py index 48cdf18d..5fd5259a 100644 --- a/backend/src/core/migration/archive_parser.py +++ b/backend/src/core/migration/archive_parser.py @@ -1,4 +1,5 @@ # #region MigrationArchiveParserModule [C:5] [TYPE Module] [SEMANTICS migration, transform, superset, migration-archive-parser] +# @defgroup Migration Module group. # @BRIEF Parse Superset export ZIP archives into normalized object catalogs for diffing. # @LAYER Domain # @RELATION DEPENDS_ON -> [LoggerModule] @@ -19,12 +20,14 @@ from ..logger import belief_scope, logger # #region MigrationArchiveParser [TYPE Class] +# @defgroup Migration Module group. # @BRIEF Extract normalized dashboards/charts/datasets metadata from ZIP archives. # @RELATION DEPENDS_ON -> [extract_objects_from_zip] # @RELATION DEPENDS_ON -> [_collect_yaml_objects] # @RELATION DEPENDS_ON -> [_normalize_object_payload] class MigrationArchiveParser: # #region extract_objects_from_zip [TYPE Function] +# @ingroup Migration # @PURPOSE: Extract object catalogs from Superset archive. # @RELATION DEPENDS_ON -> [_collect_yaml_objects] # @PRE zip_path points to a valid readable ZIP. diff --git a/backend/src/core/migration/dry_run_orchestrator.py b/backend/src/core/migration/dry_run_orchestrator.py index 1eb7210c..945a1a68 100644 --- a/backend/src/core/migration/dry_run_orchestrator.py +++ b/backend/src/core/migration/dry_run_orchestrator.py @@ -1,4 +1,5 @@ # #region MigrationDryRunOrchestratorModule [C:5] [TYPE Module] [SEMANTICS sqlalchemy, migration, orchestration, diff, migration-dry-run-service] +# @defgroup Migration Module group. # @BRIEF Compute pre-flight migration diff and risk scoring without apply. # @LAYER Domain # @RELATION DEPENDS_ON -> [SupersetClient] @@ -27,6 +28,7 @@ from .risk_assessor import build_risks, score_risks # #region MigrationDryRunService [TYPE Class] +# @defgroup Migration Module group. # @BRIEF Build deterministic diff/risk payload for migration pre-flight. # @RELATION DEPENDS_ON -> [__init__] # @RELATION DEPENDS_ON -> [run] @@ -45,6 +47,7 @@ class MigrationDryRunService: self.parser = parser or MigrationArchiveParser() # #endregion __init__ # #region run [TYPE Function] +# @ingroup Migration # @PURPOSE: Execute full dry-run computation for selected dashboards. # @RELATION DEPENDS_ON -> [_load_db_mapping] # @RELATION DEPENDS_ON -> [_accumulate_objects] diff --git a/backend/src/core/migration/risk_assessor.py b/backend/src/core/migration/risk_assessor.py index 531b07f9..2db98385 100644 --- a/backend/src/core/migration/risk_assessor.py +++ b/backend/src/core/migration/risk_assessor.py @@ -1,4 +1,5 @@ # #region RiskAssessorModule [C:5] [TYPE Module] [SEMANTICS tenacity, migration, dry-run] +# @defgroup Migration Module group. # @BRIEF Compute deterministic migration risk items and aggregate score for dry-run reporting. # @LAYER Domain # @RELATION DEPENDS_ON -> [SupersetClient] @@ -31,6 +32,7 @@ from ..superset_client import SupersetClient # #region index_by_uuid [TYPE Function] +# @ingroup Migration # @BRIEF Build UUID-index from normalized objects. # @PRE Input list items are dict-like payloads potentially containing "uuid". # @POST Returns mapping keyed by string uuid; only truthy uuid values are included. @@ -52,6 +54,7 @@ def index_by_uuid(objects: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: # #region extract_owner_identifiers [TYPE Function] +# @ingroup Migration # @BRIEF Normalize owner payloads for stable comparison. # @PRE Owners may be list payload, scalar values, or None. # @POST Returns sorted unique owner identifiers as strings. @@ -83,6 +86,7 @@ def extract_owner_identifiers(owners: Any) -> list[str]: # #region build_risks [TYPE Function] +# @ingroup Migration # @BRIEF Build risk list from computed diffs and target catalog state. # @RELATION DEPENDS_ON -> [index_by_uuid] # @RELATION DEPENDS_ON -> [extract_owner_identifiers] @@ -176,6 +180,7 @@ async def build_risks( # #region score_risks [TYPE Function] +# @ingroup Migration # @BRIEF Aggregate risk list into score and level. # @PRE risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight. # @POST Returns dict with score in [0,100], derived level, and original items. diff --git a/backend/src/core/migration_engine.py b/backend/src/core/migration_engine.py index 3b4677a0..229b9259 100644 --- a/backend/src/core/migration_engine.py +++ b/backend/src/core/migration_engine.py @@ -1,4 +1,5 @@ # #region MigrationEngineModule [C:5] [TYPE Module] [SEMANTICS migration, superset, archive, migration-engine] +# @defgroup Core Module group. # # @BRIEF Transforms Superset export ZIP archives while preserving archive integrity and patching mapped identifiers. # @LAYER Domain @@ -27,6 +28,7 @@ from .logger import belief_scope, logger # #region MigrationEngine [TYPE Class] +# @defgroup Core Module group. # @BRIEF Engine for transforming Superset export ZIPs. # @RELATION DEPENDS_ON -> [MigrationEngine] class MigrationEngine: @@ -44,6 +46,7 @@ class MigrationEngine: logger.reflect("MigrationEngine initialized") # #endregion __init__ # #region transform_zip [TYPE Function] +# @ingroup Core # @PURPOSE: Extracts ZIP, replaces database UUIDs in YAMLs, patches cross-filters, and re-packages. # @RELATION DEPENDS_ON -> [MigrationEngine] # @PARAM zip_path (str) - Path to the source ZIP file. diff --git a/backend/src/core/plugin_base.py b/backend/src/core/plugin_base.py index d881eb52..5759e994 100755 --- a/backend/src/core/plugin_base.py +++ b/backend/src/core/plugin_base.py @@ -7,6 +7,7 @@ from .logger import belief_scope # #region PluginBase [TYPE Class] [SEMANTICS plugin, interface, base, abstract] +# @defgroup Core Module group. # @BRIEF Defines the abstract base class that all plugins must implement to be recognized by the system. It enforces a common structure for plugin metadata and execution. # @LAYER Core # @PURPOSE PluginLoader scans for subclasses of PluginBase. @@ -19,6 +20,7 @@ class PluginBase(ABC): @property @abstractmethod # #region id [TYPE Function] +# @ingroup Core # @PURPOSE: Returns the unique identifier for the plugin. # @PRE Plugin instance exists. # @POST Returns string ID. @@ -31,6 +33,7 @@ class PluginBase(ABC): @property @abstractmethod # #region name [TYPE Function] +# @ingroup Core # @PURPOSE: Returns the human-readable name of the plugin. # @PRE Plugin instance exists. # @POST Returns string name. @@ -43,6 +46,7 @@ class PluginBase(ABC): @property @abstractmethod # #region description [TYPE Function] +# @ingroup Core # @PURPOSE: Returns a brief description of the plugin. # @PRE Plugin instance exists. # @POST Returns string description. @@ -55,6 +59,7 @@ class PluginBase(ABC): @property @abstractmethod # #region version [TYPE Function] +# @ingroup Core # @PURPOSE: Returns the version of the plugin. # @PRE Plugin instance exists. # @POST Returns string version. @@ -66,6 +71,7 @@ class PluginBase(ABC): # #endregion version @property # #region required_permission [TYPE Function] +# @ingroup Core # @PURPOSE: Returns the required permission string to execute this plugin. # @PRE Plugin instance exists. # @POST Returns string permission. @@ -77,6 +83,7 @@ class PluginBase(ABC): # #endregion required_permission @property # #region ui_route [TYPE Function] +# @ingroup Core # @PURPOSE: Returns the frontend route for the plugin's UI, if applicable. # @PRE Plugin instance exists. # @POST Returns string route or None. @@ -91,6 +98,7 @@ class PluginBase(ABC): # #endregion ui_route @abstractmethod # #region get_schema [TYPE Function] +# @ingroup Core # @PURPOSE: Returns the JSON schema for the plugin's input parameters. # @PRE Plugin instance exists. # @POST Returns dict schema. @@ -105,6 +113,7 @@ class PluginBase(ABC): # #endregion get_schema @abstractmethod # #region execute [TYPE Function] +# @ingroup Core # @PURPOSE: Executes the plugin's core logic. # @PARAM params (Dict[str, Any]) - Validated input parameters. # @PRE params must be a dictionary. @@ -120,6 +129,7 @@ class PluginBase(ABC): # #endregion execute # #endregion PluginBase # #region PluginConfig [TYPE Class] [SEMANTICS plugin, config, schema, pydantic] +# @defgroup Core Module group. # @BRIEF A Pydantic model used to represent the validated configuration and metadata of a loaded plugin. This object is what gets exposed to the API layer. # @LAYER Core # @PURPOSE Validated PluginConfig exposed to API layer. diff --git a/backend/src/core/plugin_loader.py b/backend/src/core/plugin_loader.py index e9c882d5..3df906fa 100755 --- a/backend/src/core/plugin_loader.py +++ b/backend/src/core/plugin_loader.py @@ -7,6 +7,7 @@ from .plugin_base import PluginBase, PluginConfig # #region PluginLoader [C:3] [TYPE Class] [SEMANTICS plugin, loader, dynamic, import] +# @defgroup Core Module group. # @BRIEF Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface. # @LAYER Core # @PURPOSE Discovers and manages available PluginBase implementations. @@ -135,6 +136,7 @@ class PluginLoader: _logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}") # #endregion _register_plugin # #region get_plugin [TYPE Function] +# @ingroup Core # @PURPOSE: Retrieves a loaded plugin instance by its ID. # @PRE plugin_id is a string. # @POST Returns plugin instance or None. @@ -148,6 +150,7 @@ class PluginLoader: return self._plugins.get(plugin_id) # #endregion get_plugin # #region get_all_plugin_configs [TYPE Function] +# @ingroup Core # @PURPOSE: Returns a list of all registered plugin configurations. # @PRE None. # @POST Returns list of all PluginConfig objects. @@ -160,6 +163,7 @@ class PluginLoader: return list(self._plugin_configs.values()) # #endregion get_all_plugin_configs # #region has_plugin [TYPE Function] +# @ingroup Core # @PURPOSE: Checks if a plugin with the given ID is registered. # @PRE plugin_id is a string. # @POST Returns True if plugin exists. diff --git a/backend/src/core/rate_limiter.py b/backend/src/core/rate_limiter.py index d3ee28c3..7e8d7da4 100644 --- a/backend/src/core/rate_limiter.py +++ b/backend/src/core/rate_limiter.py @@ -1,4 +1,5 @@ # #region RateLimiterModule [C:4] [TYPE Module] [SEMANTICS rate_limit, security, throttle, auth] +# @defgroup Core Module group. # @BRIEF Simple in-memory rate limiter for auth endpoints. # Tracks request frequency per IP and blocks excessive requests. # For production, replace with Redis-based limiter. @@ -32,6 +33,7 @@ class RateLimiter: self._bans: dict[str, float] = {} # #region is_banned [TYPE Function] +# @ingroup Core # @BRIEF Check if IP is currently banned. # @PRE ip is a valid IP string. # @POST Returns True if IP is banned and ban hasn't expired. @@ -47,6 +49,7 @@ class RateLimiter: # #endregion is_banned # #region record_attempt [TYPE Function] +# @ingroup Core # @BRIEF Record a failed attempt from an IP. # @PRE ip is a valid IP string. # @POST If MAX_ATTEMPTS exceeded within window, IP is banned. @@ -65,6 +68,7 @@ class RateLimiter: # #endregion record_attempt # #region record_success [TYPE Function] +# @ingroup Core # @BRIEF Clear attempt history on successful auth. # @PRE ip is a valid IP string. # @POST Attempt history for IP is cleared. diff --git a/backend/src/core/scheduler.py b/backend/src/core/scheduler.py index 0e89bfa1..0c96498a 100644 --- a/backend/src/core/scheduler.py +++ b/backend/src/core/scheduler.py @@ -1,4 +1,5 @@ # #region SchedulerModule [C:3] [TYPE Module] [SEMANTICS scheduler, schedule, scheduler-service] +# @defgroup Core Module group. # @BRIEF Manages scheduled tasks using APScheduler. # @LAYER Core # @RELATION DEPENDS_ON -> TaskManager @@ -15,6 +16,7 @@ from .logger import belief_scope, logger # #region SchedulerService [C:3] [TYPE Class] [SEMANTICS scheduler, service, apscheduler] +# @defgroup Core Module group. # @BRIEF Provides a service to manage scheduled backup tasks. # @RELATION DEPENDS_ON -> [ThrottledSchedulerConfigurator] class SchedulerService: @@ -30,6 +32,7 @@ class SchedulerService: self.loop = asyncio.get_event_loop() # #endregion __init__ # #region start [TYPE Function] +# @ingroup Core # @PURPOSE: Starts the background scheduler and loads initial schedules. # @PRE Scheduler should be initialized. # @POST Scheduler is running and schedules are loaded. @@ -41,6 +44,7 @@ class SchedulerService: self.load_schedules() # #endregion start # #region stop [TYPE Function] +# @ingroup Core # @PURPOSE: Stops the background scheduler. # @PRE Scheduler should be running. # @POST Scheduler is shut down. @@ -51,6 +55,7 @@ class SchedulerService: logger.info("Scheduler stopped.") # #endregion stop # #region load_schedules [C:4] [TYPE Function] [SEMANTICS scheduler,backup,translation,apscheduler] +# @ingroup Core # @BRIEF Load backup and active translation schedules from config and DB, re-registering all jobs. # @PRE config_manager must have valid configuration; database is accessible. # @POST All enabled backup jobs and active translation schedules are re-registered in APScheduler. @@ -125,6 +130,7 @@ class SchedulerService: extra={"error": str(e)}) # #endregion load_schedules # #region add_backup_job [TYPE Function] +# @ingroup Core # @PURPOSE: Adds a scheduled backup job for an environment. # @PRE env_id and cron_expression must be valid strings. # @POST A new job is added to the scheduler or replaced if it already exists. @@ -151,6 +157,7 @@ class SchedulerService: logger.error(f"Failed to add backup job for environment {env_id}: {e}") # #endregion add_backup_job # #region add_translation_job [C:4] [TYPE Function] [SEMANTICS scheduler,translation,cron,apscheduler] +# @ingroup Core # @BRIEF Register a translation schedule with APScheduler. # @PRE schedule_id, job_id, and cron_expression are valid strings. # @POST A new APScheduler job is registered or replaced if it already exists. @@ -186,6 +193,7 @@ class SchedulerService: ) # #endregion add_translation_job # #region remove_translation_job [C:4] [TYPE Function] [SEMANTICS scheduler,translation,remove,apscheduler] +# @ingroup Core # @BRIEF Remove a translation schedule from APScheduler. # @PRE schedule_id is a valid string. # @POST The APScheduler job is removed if it exists; silently ignored otherwise. @@ -233,6 +241,7 @@ class SchedulerService: ) # #endregion _trigger_backup # #region add_validation_job [C:3] [TYPE Function] [SEMANTICS scheduler,validation,cron,apscheduler] +# @ingroup Core # @BRIEF Register a validation policy schedule with APScheduler. # @PRE policy_id and cron_expression are valid strings. # @POST A new APScheduler job is registered or replaced if it already exists. @@ -265,6 +274,7 @@ class SchedulerService: ) # #endregion add_validation_job # #region remove_validation_job [C:2] [TYPE Function] [SEMANTICS scheduler,validation,remove,apscheduler] +# @ingroup Core # @BRIEF Remove a validation policy schedule from APScheduler. # @PRE policy_id is a valid string. # @POST The APScheduler job is removed if it exists; silently ignored otherwise. @@ -282,6 +292,7 @@ class SchedulerService: logger.reason(f"Validation schedule not found (already removed): {job_id_aps}") # #endregion remove_validation_job # #region reload_validation_policy [C:2] [TYPE Function] [SEMANTICS scheduler,validation,reload,apscheduler] +# @ingroup Core # @BRIEF Reload a single validation policy schedule — calls remove then add. # @PRE policy_id is a valid ValidationPolicy id with schedule data in DB. # @POST Old job is removed; new job is registered if policy is active and has schedule_days. @@ -381,6 +392,7 @@ class SchedulerService: # #endregion _trigger_validation # #endregion SchedulerService # #region ThrottledSchedulerConfigurator [C:5] [TYPE Class] [SEMANTICS scheduler, throttling, distribution] +# @defgroup Core Module group. # @BRIEF Distributes validation tasks evenly within an execution window. # @PRE Validation policies provide a finite dashboard list and a valid execution window. # @POST Produces deterministic per-dashboard run timestamps within the configured window. @@ -390,6 +402,7 @@ class SchedulerService: # @DATA_CONTRACT Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[EXT:Python:datetime]] class ThrottledSchedulerConfigurator: # #region calculate_schedule [TYPE Function] +# @ingroup Core # @PURPOSE: Calculates execution times for N tasks within a window. # @PRE window_start, window_end (time), dashboard_ids (List), current_date (date). # @POST Returns List[EXT:Python:datetime] of scheduled times. diff --git a/backend/src/core/superset_client/__init__.py b/backend/src/core/superset_client/__init__.py index 7c002032..b70d3d7d 100644 --- a/backend/src/core/superset_client/__init__.py +++ b/backend/src/core/superset_client/__init__.py @@ -1,4 +1,5 @@ # #region SupersetClientModule [C:5] [TYPE Module] [SEMANTICS superset, package, superset-client] +# @defgroup Core Module group. # @LAYER Service # @BRIEF Предоставляет высокоуровневый асинхронный клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию. # @RELATION DEPENDS_ON -> [ConfigModels] @@ -35,6 +36,7 @@ from ._user_projection import SupersetUserProjectionMixin # #region SupersetClient [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Класс-обёртка над Superset REST API, предоставляющий асинхронные методы для работы с дашбордами и датасетами. # @RELATION DEPENDS_ON -> [ConfigModels] # @RELATION DEPENDS_ON -> [AsyncAPIClient] diff --git a/backend/src/core/superset_client/_base.py b/backend/src/core/superset_client/_base.py index 2d1fbdc1..9ee4699e 100644 --- a/backend/src/core/superset_client/_base.py +++ b/backend/src/core/superset_client/_base.py @@ -1,4 +1,5 @@ # #region SupersetClientBase [C:4] [TYPE Module] [SEMANTICS superset, client, base, auth, pagination, async] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers. # @RELATION DEPENDS_ON -> [get_filename_from_headers] @@ -23,6 +24,7 @@ from ..utils.network import SupersetAPIError app_logger = cast(Any, app_logger) # #region SupersetClientBase [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Base class providing Superset client initialization, auth, pagination, and import/export plumbing. # @RELATION DEPENDS_ON -> [ConfigModels] # @RELATION DEPENDS_ON -> [AsyncAPIClient] @@ -30,6 +32,7 @@ app_logger = cast(Any, app_logger) # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для аутентификации, пагинации, импорта/экспорта. class SupersetClientBase: # #region SupersetClientInit [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент. # @RELATION DEPENDS_ON -> [Environment] # @RELATION DEPENDS_ON -> [AsyncAPIClient] @@ -63,6 +66,7 @@ class SupersetClientBase: ) # #endregion SupersetClientInit # #region SupersetClientAuthenticate [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE Authenticates the client using the configured credentials. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API для аутентификации. # @RELATION CALLS -> [AsyncAPIClient] @@ -223,6 +227,7 @@ class SupersetClientBase: return None # #endregion SupersetClientResolveTargetIdForDelete # #region SupersetClientGetAllResources [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE Fetches all resources of a given type with id, uuid, and name columns. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для получения ресурсов. # @RELATION CALLS -> [SupersetClientFetchAllPages] @@ -274,6 +279,7 @@ class SupersetClientBase: # #endregion SupersetClientGetAllResources # #region aclose [TYPE Function] +# @ingroup Core # @PURPOSE Close async HTTP transport. # @POST Underlying AsyncAPIClient session is closed. async def aclose(self) -> None: diff --git a/backend/src/core/superset_client/_charts.py b/backend/src/core/superset_client/_charts.py index 448afe9e..a79546e7 100644 --- a/backend/src/core/superset_client/_charts.py +++ b/backend/src/core/superset_client/_charts.py @@ -1,4 +1,5 @@ # #region SupersetChartsMixin [C:3] [TYPE Module] [SEMANTICS superset, chart, query, list, extract] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Chart domain mixin for SupersetClient — list, get, extract IDs from layout. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -10,11 +11,13 @@ from ..logger import belief_scope, logger as app_logger app_logger = cast(Any, app_logger) # #region SupersetChartsMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing all chart-related Superset API operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetChartsMixin: # #region SupersetClientGetChart [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches a single chart by ID. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -24,6 +27,7 @@ class SupersetChartsMixin: return cast(dict, response) # #endregion SupersetClientGetChart # #region SupersetClientGetCharts [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches all charts with pagination support. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для пагинации. # @RELATION CALLS -> [SupersetClientFetchAllPages] diff --git a/backend/src/core/superset_client/_dashboards_crud.py b/backend/src/core/superset_client/_dashboards_crud.py index 31868b7c..fcf60aab 100644 --- a/backend/src/core/superset_client/_dashboards_crud.py +++ b/backend/src/core/superset_client/_dashboards_crud.py @@ -1,4 +1,5 @@ # #region SupersetDashboardsCrudMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboard, crud, import, export] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Dashboard CRUD mixin for SupersetClient — detail, export, import, delete. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -16,6 +17,7 @@ from ..logger import belief_scope, logger as app_logger app_logger = cast(Any, app_logger) # #region SupersetDashboardsCrudMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing dashboard detail resolution, export, import, and delete operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDashboardsFiltersMixin] @@ -23,6 +25,7 @@ app_logger = cast(Any, app_logger) # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDashboardsCrudMixin: # #region SupersetClientGetDashboardDetail [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches detailed dashboard information including related charts and datasets. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. # @RELATION CALLS -> [SupersetClientGetDashboard] @@ -36,6 +39,7 @@ class SupersetDashboardsCrudMixin: charts: list[dict] = [] datasets: list[dict] = [] # #region extract_dataset_id_from_form_data [TYPE Function] +# @ingroup Core def extract_dataset_id_from_form_data( form_data: dict | None, ) -> int | None: @@ -257,6 +261,7 @@ class SupersetDashboardsCrudMixin: } # #endregion SupersetClientGetDashboardDetail # #region SupersetClientExportDashboard [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Экспортирует дашборд в виде ZIP-архива. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API для экспорта дашборда. # @RELATION CALLS -> [AsyncAPIClient] @@ -281,6 +286,7 @@ class SupersetDashboardsCrudMixin: return response.content, filename # #endregion SupersetClientExportDashboard # #region SupersetClientImportDashboard [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Импортирует дашборд из ZIP-файла. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для импорта дашборда. # @RELATION CALLS -> [SupersetClientDoImport] @@ -319,6 +325,7 @@ class SupersetDashboardsCrudMixin: return await self._do_import(file_path) # #endregion SupersetClientImportDashboard # #region SupersetClientDeleteDashboard [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Удаляет дашборд по его ID или slug. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API для удаления дашборда. # @RELATION CALLS -> [AsyncAPIClient] diff --git a/backend/src/core/superset_client/_dashboards_filters.py b/backend/src/core/superset_client/_dashboards_filters.py index 0cd1e9b7..67945401 100644 --- a/backend/src/core/superset_client/_dashboards_filters.py +++ b/backend/src/core/superset_client/_dashboards_filters.py @@ -1,4 +1,5 @@ # #region SupersetDashboardsFiltersMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboard, filter, native, advanced] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Dashboard native filter extraction mixin for SupersetClient. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -10,11 +11,13 @@ from ..logger import belief_scope, logger as app_logger app_logger = cast(Any, app_logger) # #region SupersetDashboardsFiltersMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing dashboard native filter extraction from permalink and URL state. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDashboardsFiltersMixin: # #region SupersetClientGetDashboard [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches a single dashboard by ID or slug. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -26,6 +29,7 @@ class SupersetDashboardsFiltersMixin: return cast(dict, response) # #endregion SupersetClientGetDashboard # #region SupersetClientGetDashboardPermalinkState [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Fetches stored dashboard permalink state by permalink key. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -39,6 +43,7 @@ class SupersetDashboardsFiltersMixin: return cast(dict, response) # #endregion SupersetClientGetDashboardPermalinkState # #region SupersetClientGetNativeFilterState [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Fetches stored native filter state by filter state key. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -56,6 +61,7 @@ class SupersetDashboardsFiltersMixin: return cast(dict, response) # #endregion SupersetClientGetNativeFilterState # #region SupersetClientExtractNativeFiltersFromPermalink [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Extract native filters dataMask from a permalink key. # @RELATION CALLS -> [SupersetClientGetDashboardPermalinkState] async def extract_native_filters_from_permalink(self, permalink_key: str) -> dict: @@ -85,6 +91,7 @@ class SupersetDashboardsFiltersMixin: } # #endregion SupersetClientExtractNativeFiltersFromPermalink # #region SupersetClientExtractNativeFiltersFromKey [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Extract native filters from a native_filters_key URL parameter. # @RELATION CALLS -> [SupersetClientGetNativeFilterState] async def extract_native_filters_from_key( @@ -136,6 +143,7 @@ class SupersetDashboardsFiltersMixin: } # #endregion SupersetClientExtractNativeFiltersFromKey # #region SupersetClientParseDashboardUrlForFilters [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present. # @RELATION CALLS -> [SupersetClientExtractNativeFiltersFromPermalink] # @RELATION CALLS -> [SupersetClientExtractNativeFiltersFromKey] diff --git a/backend/src/core/superset_client/_dashboards_list.py b/backend/src/core/superset_client/_dashboards_list.py index 6b388ae6..9c8f7446 100644 --- a/backend/src/core/superset_client/_dashboards_list.py +++ b/backend/src/core/superset_client/_dashboards_list.py @@ -1,4 +1,5 @@ # #region SupersetDashboardsListMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboard, list, search, filter] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Dashboard listing mixin for SupersetClient — paginated list, summary projection. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -11,12 +12,14 @@ from ..logger import belief_scope, logger as app_logger app_logger = cast(Any, app_logger) # #region SupersetDashboardsListMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing dashboard listing and summary projection operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetUserProjectionMixin] # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDashboardsListMixin: # #region SupersetClientGetDashboards [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для пагинации. # @RELATION CALLS -> [SupersetClientFetchAllPages] @@ -41,6 +44,7 @@ class SupersetDashboardsListMixin: return total_count, paginated_data # #endregion SupersetClientGetDashboards # #region SupersetClientGetDashboardsPage [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -67,6 +71,7 @@ class SupersetDashboardsListMixin: return total_count, result # #endregion SupersetClientGetDashboardsPage # #region SupersetClientGetDashboardsSummary [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches dashboard metadata optimized for the grid. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. # @RELATION CALLS -> [SupersetClientGetDashboards] @@ -122,6 +127,7 @@ class SupersetDashboardsListMixin: return result # #endregion SupersetClientGetDashboardsSummary # #region SupersetClientGetDashboardsSummaryPage [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches one page of dashboard metadata optimized for the grid. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [SupersetClientGetDashboardsPage] diff --git a/backend/src/core/superset_client/_dashboards_write.py b/backend/src/core/superset_client/_dashboards_write.py index e770c22c..6536308e 100644 --- a/backend/src/core/superset_client/_dashboards_write.py +++ b/backend/src/core/superset_client/_dashboards_write.py @@ -1,4 +1,5 @@ # #region SupersetDashboardsWriteMixin [C:4] [TYPE Module] [SEMANTICS superset, dashboard, chart, write, maintenance, banner, async] +# @defgroup Core Module group. # @BRIEF Dashboard write mixin for SupersetClient — markdown chart CRUD and layout manipulation for maintenance banners. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -25,6 +26,7 @@ app_logger = cast(Any, app_logger) # #region SupersetDashboardsWriteMixin [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing markdown chart CRUD and dashboard layout manipulation for maintenance banners. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetChartsMixin] @@ -33,6 +35,7 @@ class SupersetDashboardsWriteMixin: """Mixin for creating/updating/deleting markdown charts and manipulating dashboard layouts.""" # #region create_markdown_chart [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Create a markdown chart and return the new chart ID. # @PRE dashboard_id exists in Superset. markdown_text not empty. # @POST Returns chart_id (int). Chart is NOT yet placed in dashboard layout. @@ -72,6 +75,7 @@ class SupersetDashboardsWriteMixin: # #endregion create_markdown_chart # #region update_markdown_chart [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Update the markdown text of an existing markdown chart. # @PRE chart_id exists and is a markdown chart. # @POST Chart markdown content updated. @@ -93,6 +97,7 @@ class SupersetDashboardsWriteMixin: # #endregion update_markdown_chart # #region update_dashboard_layout [C:4] [TYPE Function] +# @ingroup Core # @BRIEF Insert a native MARKDOWN element at (0,0) with full width (12 cols), # shift existing charts down. Uses native MARKDOWN for reliable rendering. # @PRE dashboard_id exists in Superset. chart_id is used for tracking/DB. @@ -137,6 +142,7 @@ class SupersetDashboardsWriteMixin: # #endregion update_dashboard_layout # #region remove_chart_from_layout [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Remove banner markdown element from dashboard layout without deleting the chart. # @PRE dashboard_id exists. chart_id exists in layout as MARKDOWN-banner-{chart_id}. # @POST MARKDOWN element removed from position_json; items shifted up. @@ -177,6 +183,7 @@ class SupersetDashboardsWriteMixin: # #endregion remove_chart_from_layout # #region delete_chart [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Delete a chart from Superset by ID. # @PRE chart_id exists. # @POST Chart permanently deleted from Superset. @@ -194,6 +201,7 @@ class SupersetDashboardsWriteMixin: # #endregion delete_chart # #region update_banner_on_dashboard [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Update the content of a native MARKDOWN banner element on a dashboard. # @PRE dashboard_id exists. chart_id used for key lookup. # @POST MARKDOWN element content updated on dashboard. @@ -237,6 +245,7 @@ class SupersetDashboardsWriteMixin: # #region get_dashboard_layout [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Fetch the position_json layout of a dashboard. # @PRE dashboard_id exists. # @POST Returns the dashboard layout dict. diff --git a/backend/src/core/superset_client/_databases.py b/backend/src/core/superset_client/_databases.py index 201348d2..ecb378f9 100644 --- a/backend/src/core/superset_client/_databases.py +++ b/backend/src/core/superset_client/_databases.py @@ -1,4 +1,5 @@ # #region SupersetDatabasesMixin [C:3] [TYPE Module] [SEMANTICS superset, search, superset-databases-mixin] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Database domain mixin for SupersetClient — list, get, summary, by_uuid. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -9,11 +10,13 @@ from ..logger import belief_scope, logger as app_logger app_logger = cast(Any, app_logger) # #region SupersetDatabasesMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing all database-related Superset API operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDatabasesMixin: # #region SupersetClientGetDatabases [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Получает полный список баз данных. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для пагинации. # @RELATION CALLS -> [SupersetClientFetchAllPages] @@ -35,6 +38,7 @@ class SupersetDatabasesMixin: return total_count, paginated_data # #endregion SupersetClientGetDatabases # #region SupersetClientGetDatabase [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Получает информацию о конкретной базе данных по её ID. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -49,6 +53,7 @@ class SupersetDatabasesMixin: return response # #endregion SupersetClientGetDatabase # #region SupersetClientGetDatabasesSummary [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetch a summary of databases including uuid, name, and engine. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. # @RELATION CALLS -> [SupersetClientGetDatabases] @@ -61,6 +66,7 @@ class SupersetDatabasesMixin: return databases # #endregion SupersetClientGetDatabasesSummary # #region SupersetClientGetDatabaseByUuid [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Find a database by its UUID. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. # @RELATION CALLS -> [SupersetClientGetDatabases] diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index a15ac1cb..6cbff56a 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -1,4 +1,5 @@ # #region SupersetDatasetsMixin [C:3] [TYPE Module] [SEMANTICS dataset, superset, search, superset-datasets-mixin] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Dataset domain mixin for SupersetClient — list, get, detail, update. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -10,11 +11,13 @@ from ..logger import belief_scope, logger as app_logger app_logger = cast(Any, app_logger) # #region SupersetDatasetsMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing basic dataset CRUD operations (list, get, detail, update). # @RELATION DEPENDS_ON -> [SupersetClientBase] # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDatasetsMixin: # #region SupersetClientGetDatasets [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для пагинации. # @RELATION CALLS -> [SupersetClientFetchAllPages] @@ -34,6 +37,7 @@ class SupersetDatasetsMixin: return total_count, paginated_data # #endregion SupersetClientGetDatasets # #region SupersetClientGetDatasetsSummary [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. # @RELATION CALLS -> [SupersetClientGetDatasets] @@ -56,6 +60,7 @@ class SupersetDatasetsMixin: return result # #endregion SupersetClientGetDatasetsSummary # #region SupersetClientGetDatasetLinkedDashboardCount [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Fetch the number of dashboards linked to a dataset via related_objects endpoint. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -87,6 +92,7 @@ class SupersetDatasetsMixin: return 0 # #endregion SupersetClientGetDatasetLinkedDashboardCount # #region SupersetClientGetDatasetDetail [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. # @RELATION CALLS -> [SupersetClientGetDataset] @@ -212,6 +218,7 @@ class SupersetDatasetsMixin: return result # #endregion SupersetClientGetDatasetDetail # #region SupersetClientGetDataset [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Получает информацию о конкретном датасете по его ID. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] @@ -226,6 +233,7 @@ class SupersetDatasetsMixin: return response # #endregion SupersetClientGetDataset # #region SupersetClientUpdateDataset [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Обновляет данные датасета по его ID. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @RELATION CALLS -> [AsyncAPIClient] diff --git a/backend/src/core/superset_client/_datasets_preview.py b/backend/src/core/superset_client/_datasets_preview.py index 1726da41..8c0e56d6 100644 --- a/backend/src/core/superset_client/_datasets_preview.py +++ b/backend/src/core/superset_client/_datasets_preview.py @@ -1,4 +1,5 @@ # #region SupersetDatasetsPreviewMixin [C:4] [TYPE Module] [SEMANTICS superset, dataset, preview, sample, review] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Dataset preview compilation mixin for SupersetClient — build query context, compile SQL. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -13,10 +14,12 @@ from ..utils.network import SupersetAPIError # #region SupersetDatasetsPreviewMixin [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing dataset preview compilation and query context building. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDatasetsPreviewMixin: # #region SupersetClientCompileDatasetPreview [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output. # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API для компиляции SQL превью. async def compile_dataset_preview(self, dataset_id: int, template_params: dict[str, Any] | None = None, effective_filters: list[dict[str, Any]] | None = None) -> dict[str, Any]: @@ -69,6 +72,7 @@ class SupersetDatasetsPreviewMixin: raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})') # #endregion SupersetClientCompileDatasetPreview # #region SupersetClientBuildDatasetPreviewLegacyFormData [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic. def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: dict[str, Any], template_params: dict[str, Any], effective_filters: list[dict[str, Any]]) -> dict[str, Any]: with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'): @@ -99,6 +103,7 @@ class SupersetDatasetsPreviewMixin: return legacy_form_data # #endregion SupersetClientBuildDatasetPreviewLegacyFormData # #region SupersetClientBuildDatasetPreviewQueryContext [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation. def build_dataset_preview_query_context( self, diff --git a/backend/src/core/superset_client/_datasets_preview_filters.py b/backend/src/core/superset_client/_datasets_preview_filters.py index 50292a79..6bfe4aff 100644 --- a/backend/src/core/superset_client/_datasets_preview_filters.py +++ b/backend/src/core/superset_client/_datasets_preview_filters.py @@ -1,4 +1,5 @@ # #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Module] [SEMANTICS superset, dataset, preview, filter, advanced] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Filter normalization and SQL extraction helpers for dataset preview compilation. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -12,12 +13,14 @@ from ..utils.network import SupersetAPIError # #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewMixin] # @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API. class SupersetDatasetsPreviewFiltersMixin: # #region SupersetClientNormalizeEffectiveFiltersForQueryContext [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Convert execution mappings into Superset chart-data filter objects. def _normalize_effective_filters_for_query_context( self, @@ -111,6 +114,7 @@ class SupersetDatasetsPreviewFiltersMixin: } # #endregion SupersetClientNormalizeEffectiveFiltersForQueryContext # #region SupersetClientExtractCompiledSqlFromPreviewResponse [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses. def _extract_compiled_sql_from_preview_response( self, response: Any diff --git a/backend/src/core/superset_client/_layout_utils.py b/backend/src/core/superset_client/_layout_utils.py index cc43f02e..5ed185a1 100644 --- a/backend/src/core/superset_client/_layout_utils.py +++ b/backend/src/core/superset_client/_layout_utils.py @@ -1,4 +1,5 @@ # #region LayoutUtils [C:2] [TYPE Module] [SEMANTICS superset, dashboard, layout, position, json] +# @defgroup Core Module group. # @BRIEF Utility functions for manipulating Superset dashboard position_json layout. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Python:json] @@ -77,6 +78,7 @@ def _generate_banner_id(chart_id: int) -> tuple[str, str]: # #region insert_banner_markdown_at_top [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Insert a ROW + native MARKDOWN pair at the top of the dashboard grid. # Creates a ROW entry connected to GRID_ID, and a MARKDOWN child inside it. # Shifts all existing grid ROWs down by 2 rows. @@ -154,6 +156,7 @@ def update_banner_markdown_content( # #region remove_banner_from_position [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Remove the banner ROW + MARKDOWN pair from the position dict and GRID children. # Shifts remaining items up by 2 if the banner was at y=0. # @PRE position_json is a mutable dict. chart_id identifies the banner. diff --git a/backend/src/core/superset_client/_user_projection.py b/backend/src/core/superset_client/_user_projection.py index bd16c5ea..1d304004 100644 --- a/backend/src/core/superset_client/_user_projection.py +++ b/backend/src/core/superset_client/_user_projection.py @@ -1,4 +1,5 @@ # #region SupersetUserProjection [C:2] [TYPE Module] [SEMANTICS superset, user, profile, lookup, transform] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF User/owner payload normalization helpers for Superset client responses. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -6,6 +7,7 @@ from typing import Any # #region SupersetUserProjectionMixin [C:2] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing user/owner payload normalization for Superset API responses. # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetUserProjectionMixin: diff --git a/backend/src/core/superset_profile_lookup.py b/backend/src/core/superset_profile_lookup.py index ef7025a7..3721937c 100644 --- a/backend/src/core/superset_profile_lookup.py +++ b/backend/src/core/superset_profile_lookup.py @@ -1,4 +1,5 @@ # #region SupersetProfileLookup [C:5] [TYPE Module] [SEMANTICS superset, profile, account, lookup, adapter] +# @defgroup Core Module group. # # @BRIEF Provides environment-scoped Superset account lookup adapter with stable normalized output. # @LAYER Service @@ -18,6 +19,7 @@ from .utils.network import AuthenticationError, SupersetAPIError # #region SupersetAccountLookupAdapter [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Lookup Superset users and normalize candidates for profile binding. # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetProfileLookup] @@ -31,6 +33,7 @@ class SupersetAccountLookupAdapter: self.environment_id = str(environment_id or "") # #endregion __init__ # #region get_users_page [TYPE Function] +# @ingroup Core # @PURPOSE: Fetch one users page from Superset with passthrough search/sort parameters asynchronously. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API для поиска пользователей. # @PRE page_index >= 0 and page_size >= 1. @@ -176,6 +179,7 @@ class SupersetAccountLookupAdapter: } # #endregion _normalize_lookup_payload # #region normalize_user_payload [TYPE Function] +# @ingroup Core # @PURPOSE: Project raw Superset user object to canonical candidate shape. # @PRE raw_user may have heterogenous key names between Superset versions. # @POST Returns normalized candidate keys (environment_id, username, display_name, email, is_active). diff --git a/backend/src/core/task_manager/__init__.py b/backend/src/core/task_manager/__init__.py index 997ddd9a..64eb3850 100644 --- a/backend/src/core/task_manager/__init__.py +++ b/backend/src/core/task_manager/__init__.py @@ -1,4 +1,5 @@ # #region TaskManagerPackage [TYPE Module] [SEMANTICS task, manager, package, init] +# @defgroup TaskManager Module group. from .manager import TaskManager from .models import LogEntry, Task, TaskStatus diff --git a/backend/src/core/task_manager/cleanup.py b/backend/src/core/task_manager/cleanup.py index 55797c34..b575e069 100644 --- a/backend/src/core/task_manager/cleanup.py +++ b/backend/src/core/task_manager/cleanup.py @@ -1,4 +1,5 @@ # #region TaskCleanupModule [C:3] [TYPE Module] [SEMANTICS task, logs, task-cleanup-service] +# @defgroup TaskManager Module group. # @BRIEF Implements task cleanup and retention policies, including associated logs. # @LAYER Core # @RELATION DEPENDS_ON -> [TaskPersistenceService] @@ -11,6 +12,7 @@ from .persistence import TaskLogPersistenceService, TaskPersistenceService # #region TaskCleanupService [C:3] [TYPE Class] +# @defgroup TaskManager Module group. # @BRIEF Provides methods to clean up old task records and their associated logs. # @RELATION DEPENDS_ON -> [TaskPersistenceService] # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] @@ -31,6 +33,7 @@ class TaskCleanupService: self.config_manager = config_manager # #endregion __init__ # #region run_cleanup [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Deletes tasks older than the configured retention period and their logs. # @PRE Config manager has valid settings. # @POST Old tasks and their logs are deleted from persistence. @@ -56,6 +59,7 @@ class TaskCleanupService: # #endregion run_cleanup # #region delete_task_with_logs [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Delete a single task and all its associated logs. # @PRE task_id is a valid task ID. # @POST Task and all its logs are deleted. diff --git a/backend/src/core/task_manager/context.py b/backend/src/core/task_manager/context.py index e5c3b6e9..e01cdfa5 100644 --- a/backend/src/core/task_manager/context.py +++ b/backend/src/core/task_manager/context.py @@ -1,4 +1,5 @@ # #region TaskContextModule [C:5] [TYPE Module] [SEMANTICS task, execution, task-context] +# @defgroup TaskManager Module group. # @BRIEF Provides execution context passed to plugins during task execution. # @LAYER Core # @RELATION DEPENDS_ON -> [TaskLoggerModule] @@ -16,6 +17,7 @@ from .task_logger import TaskLogger # #region TaskContext [C:5] [TYPE Class] [SEMANTICS context, task, execution, plugin] +# @defgroup TaskManager Module group. # @BRIEF A container passed to plugin.execute() providing the logger and other task-specific utilities. # @INVARIANT logger is always a valid TaskLogger instance. # @PRE Constructor receives non-empty task_id, callable add_log_fn, and params mapping. @@ -73,6 +75,7 @@ class TaskContext: ) # #endregion __init__ # #region task_id [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Get the task ID. # @PRE TaskContext must be initialized. # @POST Returns the task ID string. @@ -83,6 +86,7 @@ class TaskContext: return self._task_id # #endregion task_id # #region logger [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Get the TaskLogger instance for this context. # @PRE TaskContext must be initialized. # @POST Returns the TaskLogger instance. @@ -93,6 +97,7 @@ class TaskContext: return self._logger # #endregion logger # #region params [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Get the task parameters. # @PRE TaskContext must be initialized. # @POST Returns the parameters dictionary. @@ -103,6 +108,7 @@ class TaskContext: return self._params # #endregion params # #region background_tasks [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Expose optional background task scheduler for plugins that dispatch deferred side effects. # @PRE TaskContext must be initialized. # @POST Returns BackgroundTasks-like object or None. @@ -112,6 +118,7 @@ class TaskContext: return self._background_tasks # #endregion background_tasks # #region get_param [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Get a specific parameter value with optional default. # @PRE TaskContext must be initialized. # @POST Returns parameter value or default. @@ -123,6 +130,7 @@ class TaskContext: return self._params.get(key, default) # #endregion get_param # #region create_sub_context [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Create a sub-context with a different default source. # @PRE source is a non-empty string. # @POST Returns new TaskContext with different logger source. diff --git a/backend/src/core/task_manager/event_bus.py b/backend/src/core/task_manager/event_bus.py index c3bfbf8e..f5f91808 100644 --- a/backend/src/core/task_manager/event_bus.py +++ b/backend/src/core/task_manager/event_bus.py @@ -1,4 +1,5 @@ # #region EventBusModule [C:5] [TYPE Module] [SEMANTICS event,bus,log,buffer,subscriber,flush] +# @defgroup TaskManager Module group. # @BRIEF Task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time # WebSocket observers — fully async with asyncio.Queue fan-out. # @LAYER Core @@ -30,6 +31,7 @@ QUEUE_MAXSIZE = 10000 # #region EventBus [C:5] [TYPE Class] [SEMANTICS event,bus,log,buffer,subscriber,flush] +# @defgroup TaskManager Module group. # @BRIEF Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out # for real-time observers. Fully async — uses asyncio.Queue for fan-out delivery. # @RELATION DEPENDS_ON -> [LogEntry] @@ -62,6 +64,7 @@ class EventBus: # #endregion __init__ # #region start [TYPE Function] +# @ingroup TaskManager # @BRIEF Start the background async flusher task. # @POST Flusher task is created and scheduled in the event loop. def start(self) -> None: @@ -70,6 +73,7 @@ class EventBus: # #endregion start # #region stop [TYPE Function] +# @ingroup TaskManager # @BRIEF Signal the flusher to stop and wait for it. # @POST Flusher task is cancelled and awaited. async def stop(self) -> None: @@ -83,6 +87,7 @@ class EventBus: # #endregion stop # #region async_flusher_loop [C:3] [TYPE Function] [SEMANTICS flush,background,async] +# @ingroup TaskManager # @BRIEF Background async task that periodically flushes log buffer to database. async def async_flusher_loop(self): seed_trace_id() @@ -121,6 +126,7 @@ class EventBus: # #endregion _flush_logs # #region flush_task_logs [C:3] [TYPE Function] [SEMANTICS flush,single,persistence] +# @ingroup TaskManager # @BRIEF Flush logs for a specific task immediately. # @PRE task_id exists. # @POST Task's buffered logs are written to database. @@ -136,6 +142,7 @@ class EventBus: # #endregion flush_task_logs # #region add_log [C:3] [TYPE Function] [SEMANTICS log,buffer,notify,subscriber] +# @ingroup TaskManager # @BRIEF Adds a log entry to a task buffer and notifies subscribers via async put. # @PRE Task exists. # @POST Log added to buffer and pushed to subscriber queues (if level passes filter). @@ -181,6 +188,7 @@ class EventBus: # ── Log Subscribers ── # #region subscribe_logs [C:2] [TYPE Function] [SEMANTICS subscribe,queue,realtime] +# @ingroup TaskManager # @BRIEF Subscribes to real-time logs for a task. # @POST Returns an asyncio.Queue (maxsize=10000) for log entries. async def subscribe_logs(self, task_id: str) -> asyncio.Queue: @@ -192,6 +200,7 @@ class EventBus: # #endregion subscribe_logs # #region unsubscribe_logs [C:2] [TYPE Function] [SEMANTICS unsubscribe,queue,remove] +# @ingroup TaskManager # @BRIEF Unsubscribes from real-time logs for a task. # @POST Queue removed from subscribers. def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue): @@ -205,6 +214,7 @@ class EventBus: # ── Task Status Subscribers ── # #region subscribe_status [C:2] [TYPE Function] [SEMANTICS subscribe,status,realtime] +# @ingroup TaskManager # @BRIEF Subscribes to real-time status updates for a specific task. # @POST Returns an asyncio.Queue (maxsize=10000) for Task status dicts. async def subscribe_status(self, task_id: str) -> asyncio.Queue: @@ -216,6 +226,7 @@ class EventBus: # #endregion subscribe_status # #region unsubscribe_status [C:2] [TYPE Function] [SEMANTICS unsubscribe,status,remove] +# @ingroup TaskManager # @BRIEF Unsubscribes from status updates for a task. # @POST Queue removed from status subscribers. def unsubscribe_status(self, task_id: str, queue: asyncio.Queue): @@ -227,6 +238,7 @@ class EventBus: # #endregion unsubscribe_status # #region broadcast_status [C:2] [TYPE Function] [SEMANTICS broadcast,status,notify] +# @ingroup TaskManager # @BRIEF Broadcast a task status update to all per-task and global subscribers. # @POST Status event pushed to all matching queues via async put. async def broadcast_status(self, task_id: str, task_dict: dict): @@ -253,6 +265,7 @@ class EventBus: # ── Maintenance Event Subscribers ── # #region subscribe_maintenance_events [C:2] [TYPE Function] [SEMANTICS subscribe,maintenance,events] +# @ingroup TaskManager # @BRIEF Subscribes to maintenance event updates (created/ended/banner changes). # @POST Returns an asyncio.Queue (maxsize=10000) for maintenance event dicts. async def subscribe_maintenance_events(self) -> asyncio.Queue: @@ -262,6 +275,7 @@ class EventBus: # #endregion subscribe_maintenance_events # #region unsubscribe_maintenance_events [C:2] [TYPE Function] [SEMANTICS unsubscribe,maintenance,events] +# @ingroup TaskManager # @BRIEF Unsubscribes from maintenance events. # @POST Queue removed from maintenance subscribers. def unsubscribe_maintenance_events(self, queue: asyncio.Queue): @@ -270,6 +284,7 @@ class EventBus: # #endregion unsubscribe_maintenance_events # #region broadcast_maintenance_event [C:2] [TYPE Function] [SEMANTICS broadcast,maintenance,notify] +# @ingroup TaskManager # @BRIEF Broadcast a maintenance event to all subscribers. # @POST Event pushed to all maintenance subscriber queues via async put. async def broadcast_maintenance_event(self, event: dict): @@ -285,6 +300,7 @@ class EventBus: # ── Global Task Event Subscribers ── # #region subscribe_task_events [C:2] [TYPE Function] [SEMANTICS subscribe,global,events] +# @ingroup TaskManager # @BRIEF Subscribes to ALL task events (status changes) globally. # @POST Returns an asyncio.Queue (maxsize=10000) for task event dicts. async def subscribe_task_events(self) -> asyncio.Queue: @@ -294,6 +310,7 @@ class EventBus: # #endregion subscribe_task_events # #region unsubscribe_task_events [C:2] [TYPE Function] [SEMANTICS unsubscribe,global,events] +# @ingroup TaskManager # @BRIEF Unsubscribes from global task events. # @POST Queue removed from global subscribers. def unsubscribe_task_events(self, queue: asyncio.Queue): @@ -304,6 +321,7 @@ class EventBus: # ── Log retrieval methods (sync, delegates to persistence) ── # #region get_task_logs [C:3] [TYPE Function] [SEMANTICS logs,read,persistence,backfill] +# @ingroup TaskManager # @BRIEF Retrieves logs for a specific task (from memory for running, persistence for completed). # @RELATION CALLS -> [EXT:method:TaskLogPersistenceService.get_logs] def get_task_logs( @@ -330,6 +348,7 @@ class EventBus: # #endregion get_task_logs # #region get_task_log_stats [C:2] [TYPE Function] [SEMANTICS log,stats,aggregate] +# @ingroup TaskManager # @BRIEF Get statistics about logs for a task. # @RELATION CALLS -> [EXT:method:TaskLogPersistenceService.get_log_stats] def get_task_log_stats(self, task_id: str) -> LogStats: @@ -337,6 +356,7 @@ class EventBus: # #endregion get_task_log_stats # #region get_task_log_sources [C:2] [TYPE Function] [SEMANTICS log,sources,unique] +# @ingroup TaskManager # @BRIEF Get unique sources for a task's logs. # @RELATION CALLS -> [EXT:method:TaskLogPersistenceService.get_sources] def get_task_log_sources(self, task_id: str) -> list[str]: @@ -344,6 +364,7 @@ class EventBus: # #endregion get_task_log_sources # #region delete_logs_for_tasks [C:2] [TYPE Function] [SEMANTICS log,delete,cleanup] +# @ingroup TaskManager # @BRIEF Delete logs for specified tasks. # @RELATION CALLS -> [EXT:method:TaskLogPersistenceService.delete_logs_for_tasks] def delete_logs_for_tasks(self, task_ids: list[str]) -> None: diff --git a/backend/src/core/task_manager/graph.py b/backend/src/core/task_manager/graph.py index 2abbeb35..a42c2088 100644 --- a/backend/src/core/task_manager/graph.py +++ b/backend/src/core/task_manager/graph.py @@ -1,4 +1,5 @@ # #region TaskGraphModule [C:5] [TYPE Module] [SEMANTICS task,graph,registry,persistence,crud] +# @defgroup TaskManager Module group. # @BRIEF In-memory task registry with persistence-backed hydration, pagination, and # status/plugin filtering — extracted from the monolithic TaskManager. # @LAYER Core @@ -25,6 +26,7 @@ from .persistence import TaskPersistenceService # #region TaskGraph [C:5] [TYPE Class] [SEMANTICS task,registry,crud,persistence,filter] +# @defgroup TaskManager Module group. # @BRIEF In-memory task dependency graph spanning task registry nodes, pause futures, # and persistence-backed hydration. # @RELATION DEPENDS_ON -> [Task] @@ -49,6 +51,7 @@ class TaskGraph: # #endregion __init__ # #region load_persisted_tasks [C:3] [TYPE Function] [SEMANTICS persistence,load,hydration] +# @ingroup TaskManager # @BRIEF Load persisted tasks using persistence service. # @RELATION CALLS -> [EXT:frontend:TaskPersistenceService.load_tasks] def load_persisted_tasks(self, limit: int = 100) -> None: @@ -59,6 +62,7 @@ class TaskGraph: # #endregion load_persisted_tasks # #region get_task [C:2] [TYPE Function] [SEMANTICS task,get,registry] +# @ingroup TaskManager # @BRIEF Retrieves a task by its ID. def get_task(self, task_id: str) -> Task | None: return self.tasks.get(task_id) @@ -71,6 +75,7 @@ class TaskGraph: # #endregion get_all_tasks # #region get_tasks [C:3] [TYPE Function] [SEMANTICS task,filter,paginate,sort] +# @ingroup TaskManager # @BRIEF Retrieves tasks with pagination and optional status/plugin filters. # @RELATION DEPENDS_ON -> [Task] def get_tasks( @@ -106,12 +111,14 @@ class TaskGraph: # #endregion get_tasks # #region add_task [C:2] [TYPE Function] [SEMANTICS task,register] +# @ingroup TaskManager # @BRIEF Register a task in the in-memory registry. def add_task(self, task: Task) -> None: self.tasks[task.id] = task # #endregion add_task # #region remove_tasks [C:3] [TYPE Function] [SEMANTICS task,remove,clear,persistence] +# @ingroup TaskManager # @BRIEF Remove tasks from registry and persistence, cancel futures for waiting tasks. # @RELATION CALLS -> [EXT:frontend:TaskPersistenceService.delete_tasks] def remove_tasks(self, task_ids: list[str]) -> int: diff --git a/backend/src/core/task_manager/lifecycle.py b/backend/src/core/task_manager/lifecycle.py index 8515cbb6..60bb3639 100644 --- a/backend/src/core/task_manager/lifecycle.py +++ b/backend/src/core/task_manager/lifecycle.py @@ -1,4 +1,5 @@ # #region JobLifecycleModule [C:5] [TYPE Module] [SEMANTICS task,lifecycle,state,machine,execution] +# @defgroup TaskManager Module group. # @BRIEF Task creation, execution, pause/resume, and completion transitions for plugin-backed # jobs — extracted from the monolithic TaskManager. # @LAYER Core @@ -51,6 +52,7 @@ def _task_to_dict(task: Task) -> dict: # #region JobLifecycle [C:5] [TYPE Class] [SEMANTICS task,lifecycle,execution,state,machine] +# @defgroup TaskManager Module group. # @BRIEF Encodes task creation, execution, pause/resume, and completion transitions for # plugin-backed jobs. # @RELATION DEPENDS_ON -> [TaskGraph] @@ -86,6 +88,7 @@ class JobLifecycle: # #endregion __init__ # #region create_task [C:4] [TYPE Function] [SEMANTICS task,create,persist,schedule] +# @ingroup TaskManager # @BRIEF Creates and queues a new task for execution. # @PRE Plugin with plugin_id exists. Params are valid dict. # @POST Task is created, added to registry, and scheduled for execution. @@ -210,6 +213,7 @@ class JobLifecycle: # #endregion _run_task # #region resolve_task [C:3] [TYPE Function] [SEMANTICS task,resolve,mapping,resume] +# @ingroup TaskManager # @BRIEF Resumes a task that is awaiting mapping. # @PRE Task exists and is in AWAITING_MAPPING state. # @POST Task status updated to RUNNING, params updated, execution resumed. @@ -229,6 +233,7 @@ class JobLifecycle: # #endregion resolve_task # #region wait_for_resolution [C:3] [TYPE Function] [SEMANTICS task,wait,mapping,future] +# @ingroup TaskManager # @BRIEF Pauses execution and waits for a resolution signal. # @PRE Task exists. # @POST Execution pauses until future is set. @@ -250,6 +255,7 @@ class JobLifecycle: # #endregion wait_for_resolution # #region wait_for_input [C:3] [TYPE Function] [SEMANTICS task,wait,input,future] +# @ingroup TaskManager # @BRIEF Pauses execution and waits for user input. # @PRE Task exists. # @POST Execution pauses until future is set via resume_task_with_password. @@ -268,6 +274,7 @@ class JobLifecycle: # #endregion wait_for_input # #region await_input [C:3] [TYPE Function] [SEMANTICS task,input,pause,state] +# @ingroup TaskManager # @BRIEF Transition a task to AWAITING_INPUT state with input request. # @PRE Task exists and is in RUNNING state. # @POST Task status changed to AWAITING_INPUT, input_request set, persisted. @@ -298,6 +305,7 @@ class JobLifecycle: # #endregion await_input # #region resume_task_with_password [C:3] [TYPE Function] [SEMANTICS task,resume,password,input] +# @ingroup TaskManager # @BRIEF Resume a task that is awaiting input with provided passwords. # @PRE Task exists and is in AWAITING_INPUT state. # @POST Task status changed to RUNNING, passwords injected, task resumed. @@ -334,6 +342,7 @@ class JobLifecycle: # #endregion resume_task_with_password # #region subscribe_dataset_events [C:2] [TYPE Function] [SEMANTICS dataset,subscribe,events,websocket] +# @ingroup TaskManager # @BRIEF Subscribe to dataset.updated events for a specific environment. async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue: queue = asyncio.Queue() @@ -344,6 +353,7 @@ class JobLifecycle: # #endregion subscribe_dataset_events # #region unsubscribe_dataset_events [C:2] [TYPE Function] [SEMANTICS dataset,unsubscribe,events] +# @ingroup TaskManager # @BRIEF Unsubscribe from dataset.updated events for a specific environment. def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue): if env_id in self._dataset_subscribers: diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index eeee7b68..9e175deb 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -1,4 +1,5 @@ # #region TaskManagerModule [C:5] [TYPE Module] [SEMANTICS task, schedule, execution, task-manager] +# @defgroup TaskManager Module group. # @BRIEF Thin facade composing TaskGraph (registry), EventBus (log/pub-sub), and JobLifecycle # (state machine) into a single TaskManager interface for backward compatibility. # @LAYER Core @@ -44,6 +45,7 @@ from src.core.task_manager.persistence import TaskLogPersistenceService, TaskPer # #region TaskManager [C:5] [TYPE Class] [SEMANTICS task, manager, lifecycle, execution, state] +# @defgroup TaskManager Module group. # @BRIEF Facade composing TaskGraph, EventBus, and JobLifecycle into a single interface. # @LAYER Core # @RELATION DEPENDS_ON -> [TaskPersistenceService] @@ -149,6 +151,7 @@ class TaskManager: # ── Task CRUD delegates to TaskGraph ── # #region get_task [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Retrieves a task by its ID. def get_task(self, task_id: str) -> Task | None: return self.graph.get_task(task_id) @@ -161,6 +164,7 @@ class TaskManager: # #endregion get_all_tasks # #region get_tasks [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Retrieves tasks with pagination and optional status filter. def get_tasks( self, @@ -174,12 +178,14 @@ class TaskManager: # #endregion get_tasks # #region load_persisted_tasks [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Load persisted tasks using persistence service. def load_persisted_tasks(self) -> None: self.graph.load_persisted_tasks(limit=100) # #endregion load_persisted_tasks # #region clear_tasks [TYPE Function] [C:4] +# @ingroup TaskManager # @BRIEF Clears tasks based on status filter (also deletes associated logs). # @SIDE_EFFECT Removes tasks from registry and persistence; cancels futures. def clear_tasks(self, status: TaskStatus | None = None) -> int: @@ -211,6 +217,7 @@ class TaskManager: # ── Log delegates to EventBus ── # #region get_task_logs [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Retrieves logs for a specific task (from memory or persistence). def get_task_logs( self, task_id: str, log_filter: LogFilter | None = None @@ -224,12 +231,14 @@ class TaskManager: # #endregion get_task_logs # #region get_task_log_stats [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Get statistics about logs for a task. def get_task_log_stats(self, task_id: str) -> LogStats: return self.event_bus.get_task_log_stats(task_id) # #endregion get_task_log_stats # #region get_task_log_sources [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Get unique sources for a task's logs. def get_task_log_sources(self, task_id: str) -> list[str]: return self.event_bus.get_task_log_sources(task_id) @@ -238,12 +247,14 @@ class TaskManager: # ── Subscription delegates to EventBus ── # #region subscribe_logs [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Subscribes to real-time logs for a task. async def subscribe_logs(self, task_id: str) -> asyncio.Queue: return await self.event_bus.subscribe_logs(task_id) # #endregion subscribe_logs # #region unsubscribe_logs [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Unsubscribes from real-time logs for a task. def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue): self.event_bus.unsubscribe_logs(task_id, queue) @@ -252,24 +263,28 @@ class TaskManager: # ── Status subscribers ── # #region subscribe_status [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Subscribes to real-time status updates for a task. async def subscribe_status(self, task_id: str) -> asyncio.Queue: return await self.event_bus.subscribe_status(task_id) # #endregion subscribe_status # #region unsubscribe_status [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Unsubscribes from status updates for a task. def unsubscribe_status(self, task_id: str, queue: asyncio.Queue): self.event_bus.unsubscribe_status(task_id, queue) # #endregion unsubscribe_status # #region subscribe_task_events [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Subscribes to global task events (all task status changes). async def subscribe_task_events(self) -> asyncio.Queue: return await self.event_bus.subscribe_task_events() # #endregion subscribe_task_events # #region unsubscribe_task_events [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Unsubscribes from global task events. def unsubscribe_task_events(self, queue: asyncio.Queue): self.event_bus.unsubscribe_task_events(queue) @@ -278,6 +293,7 @@ class TaskManager: # ── Lifecycle delegates to JobLifecycle ── # #region create_task [TYPE Function] [C:4] +# @ingroup TaskManager # @BRIEF Creates and queues a new task for execution. async def create_task( self, plugin_id: str, params: dict[str, Any], user_id: str | None = None @@ -309,6 +325,7 @@ class TaskManager: # #endregion _run_task # #region cancel_task [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Cancel a running task by ID. # @PRE Task must be currently tracked as running. # @POST Task is cancelled and removed from tracking dict. @@ -322,24 +339,28 @@ class TaskManager: # #endregion cancel_task # #region resolve_task [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Resumes a task that is awaiting mapping. async def resolve_task(self, task_id: str, resolution_params: dict[str, Any]): await self.lifecycle.resolve_task(task_id, resolution_params) # #endregion resolve_task # #region wait_for_resolution [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Pauses execution and waits for a resolution signal. async def wait_for_resolution(self, task_id: str): await self.lifecycle.wait_for_resolution(task_id) # #endregion wait_for_resolution # #region wait_for_input [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Pauses execution and waits for user input. async def wait_for_input(self, task_id: str): await self.lifecycle.wait_for_input(task_id) # #endregion wait_for_input # #region await_input [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Transition a task to AWAITING_INPUT state with input request. async def await_input(self, task_id: str, input_request: dict[str, Any]) -> None: await self.lifecycle.await_input( @@ -349,6 +370,7 @@ class TaskManager: # #endregion await_input # #region resume_task_with_password [TYPE Function] [C:3] +# @ingroup TaskManager # @BRIEF Resume a task that is awaiting input with provided passwords. async def resume_task_with_password( self, task_id: str, passwords: dict[str, str] @@ -362,18 +384,21 @@ class TaskManager: # ── Maintenance event delegates to EventBus ── # #region subscribe_maintenance_events [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Subscribes to global maintenance events. async def subscribe_maintenance_events(self) -> asyncio.Queue: return await self.event_bus.subscribe_maintenance_events() # #endregion subscribe_maintenance_events # #region unsubscribe_maintenance_events [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Unsubscribes from global maintenance events. def unsubscribe_maintenance_events(self, queue: asyncio.Queue): self.event_bus.unsubscribe_maintenance_events(queue) # #endregion unsubscribe_maintenance_events # #region broadcast_maintenance_event [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Broadcast a maintenance event to all subscribers. async def broadcast_maintenance_event(self, event: dict): await self.event_bus.broadcast_maintenance_event(event) @@ -382,12 +407,14 @@ class TaskManager: # ── Dataset event delegates to JobLifecycle ── # #region subscribe_dataset_events [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Subscribe to dataset.updated events for an environment. async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue: return await self.lifecycle.subscribe_dataset_events(env_id) # #endregion subscribe_dataset_events # #region unsubscribe_dataset_events [TYPE Function] [C:2] +# @ingroup TaskManager # @BRIEF Unsubscribe from dataset.updated events. def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue): self.lifecycle.unsubscribe_dataset_events(env_id, queue) diff --git a/backend/src/core/task_manager/models.py b/backend/src/core/task_manager/models.py index 86d9c649..78e5e0bd 100644 --- a/backend/src/core/task_manager/models.py +++ b/backend/src/core/task_manager/models.py @@ -1,4 +1,5 @@ # #region TaskManagerModels [C:5] [TYPE Module] [SEMANTICS pydantic, task, model, validate, task-status] +# @defgroup TaskManager Module group. # @BRIEF Defines the data models and enumerations used by the Task Manager. # @LAYER Domain # @RELATION CALLED_BY -> [TaskManager] @@ -18,6 +19,7 @@ from pydantic import BaseModel, ConfigDict, Field # #region TaskStatus [TYPE Enum] +# @ingroup TaskManager class TaskStatus(str, Enum): PENDING = "PENDING" RUNNING = "RUNNING" @@ -27,6 +29,7 @@ class TaskStatus(str, Enum): AWAITING_INPUT = "AWAITING_INPUT" # #endregion TaskStatus # #region LogLevel [TYPE Enum] +# @ingroup TaskManager class LogLevel(str, Enum): DEBUG = "DEBUG" INFO = "INFO" @@ -34,6 +37,7 @@ class LogLevel(str, Enum): ERROR = "ERROR" # #endregion LogLevel # #region LogEntry [C:2] [TYPE Class] +# @defgroup TaskManager Module group. # @BRIEF A Pydantic model representing a single, structured log entry associated with a task. # # { @@ -55,6 +59,7 @@ class LogEntry(BaseModel): ) # #endregion LogEntry # #region TaskLog [C:3] [TYPE Class] [SEMANTICS task, log, persistent, pydantic] +# @defgroup TaskManager Module group. # @BRIEF A Pydantic model representing a persisted log entry from the database. # @RELATION DEPENDS_ON -> [TaskLogRecord] class TaskLog(BaseModel): @@ -68,6 +73,7 @@ class TaskLog(BaseModel): model_config = ConfigDict(from_attributes=True) # #endregion TaskLog # #region LogFilter [TYPE Class] +# @defgroup TaskManager Module group. class LogFilter(BaseModel): level: str | None = None # Filter by log level source: str | None = None # Filter by source component @@ -84,6 +90,7 @@ class LogStats(BaseModel): by_source: dict[str, int] # {"plugin": 5, "superset_api": 7} # #endregion LogStats # #region Task [C:3] [TYPE Class] [SEMANTICS task, job, execution, state, pydantic] +# @defgroup TaskManager Module group. # @BRIEF A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs. # @RELATION DEPENDS_ON -> [TaskStatus] # @RELATION DEPENDS_ON -> [LogEntry] diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index daad6c57..7debb001 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -1,4 +1,5 @@ # #region TaskPersistenceModule [C:5] [TYPE Module] [SEMANTICS sqlalchemy, task, search, task-persistence-service] +# @defgroup TaskManager Module group. # @BRIEF Handles the persistence of tasks using SQLAlchemy and the tasks.db database. # @LAYER Core # @PRE Tasks database must be initialized with TaskRecord and TaskLogRecord schemas. @@ -23,6 +24,7 @@ from .models import LogEntry, LogFilter, LogStats, Task, TaskLog, TaskStatus # #region TaskPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, sqlalchemy] +# @defgroup TaskManager Module group. # @BRIEF Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models. # @PRE TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available. # @POST Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows. @@ -137,6 +139,7 @@ class TaskPersistenceService: pass # #endregion __init__ # #region persist_task [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Persists or updates a single task in the database. # @PRE isinstance(task, Task) # @POST Task record created or updated in database. @@ -200,6 +203,7 @@ class TaskPersistenceService: session.close() # #endregion persist_task # #region persist_tasks [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Persists multiple tasks. # @PRE isinstance(tasks, list) # @POST All tasks in list are persisted. @@ -211,6 +215,7 @@ class TaskPersistenceService: self.persist_task(task) # #endregion persist_tasks # #region load_tasks [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Loads tasks from the database. # @PRE limit is an integer. # @POST Returns list of Task objects. @@ -269,6 +274,7 @@ class TaskPersistenceService: session.close() # #endregion load_tasks # #region delete_tasks [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Deletes specific tasks from the database. # @PRE task_ids is a list of strings. # @POST Specified task records deleted from database. @@ -293,6 +299,7 @@ class TaskPersistenceService: # #endregion delete_tasks # #endregion TaskPersistenceService # #region TaskLogPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, log, sqlalchemy] +# @defgroup TaskManager Module group. # @BRIEF Provides methods to store, query, summarize, and delete task log rows in the task_logs table. # @PRE TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries. # @POST add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers. @@ -329,6 +336,7 @@ class TaskLogPersistenceService: pass # #endregion __init__ # #region add_logs [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Batch insert log entries for a task. # @PRE logs is a list of LogEntry objects. # @POST All logs inserted into task_logs table. @@ -363,6 +371,7 @@ class TaskLogPersistenceService: session.close() # #endregion add_logs # #region get_logs [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Query logs for a task with filtering and pagination. # @PRE task_id is a valid task ID. # @POST Returns list of TaskLog objects matching filters. @@ -418,6 +427,7 @@ class TaskLogPersistenceService: session.close() # #endregion get_logs # #region get_log_stats [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Get statistics about logs for a task. # @PRE task_id is a valid task ID. # @POST Returns LogStats with counts by level and source. @@ -462,6 +472,7 @@ class TaskLogPersistenceService: session.close() # #endregion get_log_stats # #region get_sources [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Get unique sources for a task's logs. # @PRE task_id is a valid task ID. # @POST Returns list of unique source strings. @@ -486,6 +497,7 @@ class TaskLogPersistenceService: session.close() # #endregion get_sources # #region delete_logs_for_task [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Delete all logs for a specific task. # @PRE task_id is a valid task ID. # @POST All logs for the task are deleted. @@ -509,6 +521,7 @@ class TaskLogPersistenceService: session.close() # #endregion delete_logs_for_task # #region delete_logs_for_tasks [TYPE Function] [C:3] +# @ingroup TaskManager # @PURPOSE: Delete all logs for multiple tasks. # @PRE task_ids is a list of task IDs. # @POST All logs for the tasks are deleted. diff --git a/backend/src/core/task_manager/task_logger.py b/backend/src/core/task_manager/task_logger.py index 4a77f051..d174fea2 100644 --- a/backend/src/core/task_manager/task_logger.py +++ b/backend/src/core/task_manager/task_logger.py @@ -1,4 +1,5 @@ # #region TaskLoggerModule [C:2] [TYPE Module] [SEMANTICS task, logger, log, streaming] +# @defgroup TaskManager Module group. # @BRIEF Provides a dedicated logger for tasks with automatic source attribution. # @LAYER Core # @RELATION DEPENDS_ON -> [TaskManager] @@ -10,6 +11,7 @@ from typing import Any # #region TaskLogger [C:2] [TYPE Class] [SEMANTICS logger, task, source, attribution] +# @defgroup TaskManager Module group. # @BRIEF A wrapper around TaskManager._add_log that carries task_id and source context. # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [EventBus] @@ -54,6 +56,7 @@ class TaskLogger: self._default_source = source # #endregion __init__ # #region with_source [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Create a sub-logger with a different default source. # @PRE source is a non-empty string. # @POST Returns new TaskLogger with the same task_id but different source. @@ -99,6 +102,7 @@ class TaskLogger: pass # #endregion _log # #region debug [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Log a DEBUG level message. # @PRE message is a string. # @POST Log entry added via internally with DEBUG level. @@ -114,6 +118,7 @@ class TaskLogger: self._log("DEBUG", message, source, metadata) # #endregion debug # #region info [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Log an INFO level message. # @PRE message is a string. # @POST Log entry added internally with INFO level. @@ -129,6 +134,7 @@ class TaskLogger: self._log("INFO", message, source, metadata) # #endregion info # #region warning [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Log a WARNING level message. # @PRE message is a string. # @POST Log entry added internally with WARNING level. @@ -144,6 +150,7 @@ class TaskLogger: self._log("WARNING", message, source, metadata) # #endregion warning # #region error [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Log an ERROR level message. # @PRE message is a string. # @POST Log entry added internally with ERROR level. @@ -159,6 +166,7 @@ class TaskLogger: self._log("ERROR", message, source, metadata) # #endregion error # #region progress [TYPE Function] +# @ingroup TaskManager # @PURPOSE: Log a progress update with percentage. # @PRE percent is between 0 and 100. # @POST Log entry with progress metadata added. diff --git a/backend/src/core/timezone.py b/backend/src/core/timezone.py index 31f3258b..b2472bfd 100644 --- a/backend/src/core/timezone.py +++ b/backend/src/core/timezone.py @@ -1,4 +1,5 @@ # #region AppTimezone [C:3] [TYPE Module] [SEMANTICS core,timezone,utilities,datetime] +# @defgroup Core Module group. # @BRIEF Application-level timezone utilities. Reads APP_TIMEZONE from env (default Europe/Moscow) # and provides helpers for converting UTC datetimes to the configured timezone. # @RELATION CALLED_BY -> [ConfigManager] diff --git a/backend/src/core/utils/__init__.py b/backend/src/core/utils/__init__.py index 57d3eb62..bfbfce73 100644 --- a/backend/src/core/utils/__init__.py +++ b/backend/src/core/utils/__init__.py @@ -1,3 +1,4 @@ # #region CoreUtils [TYPE Package] [SEMANTICS utils, package, init] +# @ingroup Core # @BRIEF Shared utility package root. # #endregion CoreUtils diff --git a/backend/src/core/utils/async_network.py b/backend/src/core/utils/async_network.py index 7f5d6bec..efe32ddc 100644 --- a/backend/src/core/utils/async_network.py +++ b/backend/src/core/utils/async_network.py @@ -1,4 +1,5 @@ # #region AsyncNetworkModule [C:5] [TYPE Module] [SEMANTICS auth, superset, token, async-client] +# @defgroup Core Module group. # # @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login. # @LAYER Infrastructure @@ -33,6 +34,7 @@ from .network import ( # #region AsyncAPIClient [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache and optional semaphore. # @PRE config contains base_url and auth payload. # @POST Client is ready for async request/authentication flow. Connection pool initialised. @@ -50,6 +52,7 @@ class AsyncAPIClient: DEFAULT_TIMEOUT = 30 _auth_locks: dict[tuple[str, str, bool], asyncio.Lock] = {} # #region AsyncAPIClient.__init__ [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Initialize async API client for one environment with optional shared semaphore. # @PRE config contains base_url and auth payload. # @POST Client is ready for async request/authentication flow. @@ -112,6 +115,7 @@ class AsyncAPIClient: return created_lock # #endregion AsyncAPIClient._get_auth_lock # #region AsyncAPIClient.authenticate [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Authenticate against Superset and cache access/csrf tokens. # @POST Client tokens are populated and reusable across requests. # @SIDE_EFFECT Performs network requests to Superset authentication endpoints. @@ -196,6 +200,7 @@ class AsyncAPIClient: raise NetworkError(f"Network or parsing error during authentication: {exc}") from exc # #endregion AsyncAPIClient.authenticate # #region AsyncAPIClient.get_headers [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Return authenticated Superset headers for async requests. # @POST Headers include Authorization and CSRF tokens. # @RELATION CALLS -> [AsyncAPIClient.authenticate] @@ -210,6 +215,7 @@ class AsyncAPIClient: } # #endregion AsyncAPIClient.get_headers # #region AsyncAPIClient.request [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Perform one authenticated async Superset API request with optional semaphore. # @POST Returns JSON payload (dict) or raw httpx.Response when raw_response=true. # @SIDE_EFFECT Performs network I/O. Acquires/releases semaphore if configured. @@ -294,6 +300,7 @@ class AsyncAPIClient: # #endregion AsyncAPIClient.request # #region AsyncAPIClient._handle_http_error [TYPE Function] [C:3] +# @ingroup Core def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None: with belief_scope("AsyncAPIClient._handle_http_error"): status_code = exc.response.status_code @@ -315,6 +322,7 @@ class AsyncAPIClient: raise SupersetAPIError(f"API Error {status_code}: {exc.response.text}") from exc # #endregion AsyncAPIClient._handle_http_error # #region AsyncAPIClient._is_dashboard_endpoint [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation. # @POST Returns true only for dashboard-specific endpoints. def _is_dashboard_endpoint(self, endpoint: str) -> bool: @@ -331,6 +339,7 @@ class AsyncAPIClient: return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard" # #endregion AsyncAPIClient._is_dashboard_endpoint # #region AsyncAPIClient._handle_network_error [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Translate generic httpx errors into NetworkError. # @POST Raises NetworkError with URL context. # @DATA_CONTRACT Input[httpx.HTTPError] -> NetworkError @@ -346,6 +355,7 @@ class AsyncAPIClient: raise NetworkError(message, url=url) from exc # #endregion AsyncAPIClient._handle_network_error # #region AsyncAPIClient.fetch_paginated_count [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetch total count of items for a paginated endpoint. # @POST Returns integer count from API response. # @RELATION CALLS -> [AsyncAPIClient.request] @@ -365,6 +375,7 @@ class AsyncAPIClient: # #endregion AsyncAPIClient.fetch_paginated_count # #region AsyncAPIClient.fetch_paginated_data [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Fetch all pages of a paginated endpoint with automatic page iteration. # @POST Returns combined list of all items across pages. # @SIDE_EFFECT Performs multiple sequential GET requests to iterate pages. @@ -414,6 +425,7 @@ class AsyncAPIClient: # #endregion AsyncAPIClient.fetch_paginated_data # #region AsyncAPIClient.upload_file [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Upload a file via multipart/form-data POST with optional extra data fields. # @PRE endpoint is valid relative Superset API endpoint. file_info contains file_obj and file_name. # @POST Returns parsed JSON API response. @@ -479,6 +491,7 @@ class AsyncAPIClient: # #endregion AsyncAPIClient.upload_file # #region AsyncAPIClient.aclose [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Close underlying httpx client. # @POST Client resources are released. # @SIDE_EFFECT Closes network connections. diff --git a/backend/src/core/utils/client_registry.py b/backend/src/core/utils/client_registry.py index f824c235..67a2d282 100644 --- a/backend/src/core/utils/client_registry.py +++ b/backend/src/core/utils/client_registry.py @@ -1,4 +1,5 @@ # #region SupersetClientRegistryModule [C:4] [TYPE Module] [SEMANTICS async, superset, client, registry, semaphore] +# @defgroup Core Module group. # @BRIEF Singleton registry of long-lived async Superset clients per environment (env_id). # Stores shared httpx.AsyncClient + shared asyncio.Semaphore + asyncio.Lock for auth refresh. # @LAYER Infrastructure @@ -49,6 +50,7 @@ def _build_env_id(env: Any) -> str: # #region get_client [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Get or create an AsyncAPIClient for the given environment config. # Internal — consumers should use get_superset_client. # @PRE env is a valid Environment model or dict with base_url and auth. @@ -112,6 +114,7 @@ async def get_client( # #region get_superset_client [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Get or create a shared SupersetClient for the given environment. # Returns a lightweight SupersetClient wrapping the shared AsyncAPIClient. # All callers across the project should use this instead of SupersetClient(env). @@ -138,6 +141,7 @@ async def get_superset_client( # #region get_semaphore [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Return the shared semaphore for the given env_config. # @PRE Client must have been created via get_client first. # @POST Returns asyncio.Semaphore instance. @@ -154,6 +158,7 @@ async def get_semaphore(env: Any) -> asyncio.Semaphore: # #region get_auth_lock [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Return the shared auth lock for the given environment. # @PRE Client must have been created via get_client first. # @POST Returns asyncio.Lock instance for serializing auth refresh. @@ -170,6 +175,7 @@ async def get_auth_lock(env: Any) -> asyncio.Lock: # #region shutdown [C:3] [TYPE Function] +# @ingroup Core # @BRIEF Close all async clients and clear the registry. # @PRE Application is shutting down. # @POST All httpx.AsyncClient instances are closed. Registry cleared. diff --git a/backend/src/core/utils/dataset_mapper.py b/backend/src/core/utils/dataset_mapper.py index cec91af6..1128532e 100644 --- a/backend/src/core/utils/dataset_mapper.py +++ b/backend/src/core/utils/dataset_mapper.py @@ -1,4 +1,5 @@ # #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, superset, dataset-mapper, sqllab] +# @defgroup Core Module group. # # @BRIEF Этот модуль отвечает за обновление метаданных (verbose_name) в датасетах Superset, # извлекая их из Superset SQL Lab (любая БД, подключённая к Superset) или XLSX-файлов. @@ -14,6 +15,7 @@ from ..logger import belief_scope, logger as app_logger # #region DatasetMapper [TYPE Class] +# @defgroup Core Module group. # @BRIEF Класс для маппинга и обновления verbose_name в датасетах Superset. class DatasetMapper: # #region __init__ [TYPE Function] @@ -24,6 +26,7 @@ class DatasetMapper: # #endregion __init__ # #region get_sqllab_mappings [TYPE Function] +# @ingroup Core # @PURPOSE: Извлекает маппинги column_name -> verbose_name через SQL Lab Superset. # @PRE sqllab_executor должен быть инициализирован с database_id. # @PRE dataset_id должен существовать в Superset. @@ -88,6 +91,7 @@ class DatasetMapper: # #endregion get_sqllab_mappings # #region load_excel_mappings [TYPE Function] +# @ingroup Core # @PURPOSE: Загружает маппинги column_name -> verbose_name из XLSX файла. # @PRE file_path должен указывать на существующий XLSX файл. # @POST Возвращается словарь с маппингами из файла. @@ -108,6 +112,7 @@ class DatasetMapper: # #endregion load_excel_mappings # #region run_mapping [TYPE Function] +# @ingroup Core # @PURPOSE: Основная функция для выполнения маппинга и обновления verbose_name датасета в Superset. # @PRE superset_client должен быть авторизован. # @PRE dataset_id должен быть существующим ID в Superset. diff --git a/backend/src/core/utils/executors.py b/backend/src/core/utils/executors.py index 126e0bcf..3bb4f3f3 100644 --- a/backend/src/core/utils/executors.py +++ b/backend/src/core/utils/executors.py @@ -1,4 +1,5 @@ # #region BlockingExecutorsModule [C:4] [TYPE Module] [SEMANTICS async, blocking, executors, run_blocking, threadpool] +# @defgroup Core Module group. # @BRIEF Named bounded executors for sync DB/file/git work. All production blocking operations use # loop.run_in_executor via the run_blocking helper instead of default asyncio.to_thread. # @LAYER Infrastructure @@ -32,6 +33,7 @@ _DEFAULT_QUEUE_TIMEOUT = 30.0 # #region init_executors [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Initialize named executors with given config. Called once at application startup. # @PRE event loop is running. # @POST Executors and semaphores are ready for use. @@ -61,6 +63,7 @@ def init_executors( # #region shutdown_executors [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Shut down all executors. Called at application shutdown. # @POST All executors are shut down, pending futures cancelled. # @SIDE_EFFECT Waits for running tasks up to timeout. @@ -94,6 +97,7 @@ def _get_executor(kind: str) -> tuple[ThreadPoolExecutor, asyncio.Semaphore | No # #region run_blocking [C:4] [TYPE Function] +# @ingroup Core # @BRIEF Execute a blocking function in a named bounded executor. # @PRE init_executors has been called. kind is one of 'db', 'file', 'git'. # @POST fn(*args) executed in named executor. Returns result or raises timeout/exception. @@ -137,6 +141,7 @@ async def run_blocking( # #region run_cpu_blocking [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Execute CPU-bound work in default executor (no bounded pool needed). # @POST fn(*args) executed in default thread pool. # @SIDE_EFFECT Runs function in thread pool. Does not acquire semaphore. diff --git a/backend/src/core/utils/fileio.py b/backend/src/core/utils/fileio.py index 2c899289..c9d6a30a 100644 --- a/backend/src/core/utils/fileio.py +++ b/backend/src/core/utils/fileio.py @@ -1,4 +1,5 @@ # #region FileIO [C:3] [TYPE Module] [SEMANTICS fileio, archive, zip, yaml, utility] +# @defgroup Core Module group. # @BRIEF Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [LoggerModule] @@ -20,11 +21,13 @@ from src.core.logger import belief_scope, logger as app_logger # #region InvalidZipFormatError [TYPE Class] +# @defgroup Core Module group. # @BRIEF Exception raised when a file is not a valid ZIP archive. class InvalidZipFormatError(Exception): pass # #endregion InvalidZipFormatError # #region create_temp_file [TYPE Function] +# @ingroup Core # @BRIEF Контекстный менеджер для создания временного файла или директории с гарантированным удалением. # @PRE suffix должен быть строкой, определяющей тип ресурса. # @POST Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста. @@ -61,6 +64,7 @@ def create_temp_file(content: bytes | None = None, suffix: str = ".zip", mode: s app_logger.error("[create_temp_file][Failure] Error during cleanup of %s: %s", resource_path, e) # #endregion create_temp_file # #region remove_empty_directories [TYPE Function] +# @ingroup Core # @BRIEF Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути. # @PRE root_dir должен быть путем к существующей директории. # @POST Все пустые поддиректории удалены, возвращено их количество. @@ -83,6 +87,7 @@ def remove_empty_directories(root_dir: str) -> int: return removed_count # #endregion remove_empty_directories # #region read_dashboard_from_disk [TYPE Function] +# @ingroup Core # @BRIEF Читает бинарное содержимое файла с диска. # @PRE file_path должен указывать на существующий файл. # @POST Возвращает байты содержимого и имя файла. @@ -97,6 +102,7 @@ def read_dashboard_from_disk(file_path: str) -> tuple[bytes, str]: return content, path.name # #endregion read_dashboard_from_disk # #region calculate_crc32 [TYPE Function] +# @ingroup Core # @BRIEF Вычисляет контрольную сумму CRC32 для файла. # @PRE file_path должен быть объектом Path к существующему файлу. # @POST Возвращает 8-значную hex-строку CRC32. @@ -106,6 +112,7 @@ def calculate_crc32(file_path: Path) -> str: return f"{crc32_value:08x}" # #endregion calculate_crc32 # #region RetentionPolicy [TYPE DataClass] +# @ingroup Core # @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные). @dataclass class RetentionPolicy: @@ -114,6 +121,7 @@ class RetentionPolicy: monthly: int = 12 # #endregion RetentionPolicy # #region archive_exports [TYPE Function] +# @ingroup Core # @BRIEF Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию. # @PRE output_dir должен быть путем к существующей директории. # @POST Старые или дублирующиеся архивы удалены согласно политике. @@ -189,6 +197,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool app_logger.error("[archive_exports][Failure] Failed to remove %s: %s", file_path, e) # #endregion archive_exports # #region apply_retention_policy [TYPE Function] +# @ingroup Core # @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить. # @PRE files_with_dates is a list of (Path, date) tuples. # @POST Returns a set of files to keep. @@ -220,6 +229,7 @@ def apply_retention_policy(files_with_dates: list[tuple[Path, date]], policy: Re return files_to_keep # #endregion apply_retention_policy # #region save_and_unpack_dashboard [TYPE Function] +# @ingroup Core # @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его. # @PRE zip_content должен быть байтами валидного ZIP-архива. # @POST ZIP-файл сохранен, и если unpack=True, он распакован в output_dir. @@ -244,6 +254,7 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: str | Path, unpack raise InvalidZipFormatError(f"Invalid ZIP file: {e}") from e # #endregion save_and_unpack_dashboard # #region update_yamls [TYPE Function] +# @ingroup Core # @BRIEF Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex. # @PRE path должен быть существующей директорией. # @POST Все YAML файлы в директории обновлены согласно переданным параметрам. @@ -303,6 +314,7 @@ def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_ pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)' # #region replacer [TYPE Function] +# @ingroup Core # @PURPOSE: Функция замены, сохраняющая кавычки если они были. # @PRE match должен быть объектом совпадения регулярного выражения. # @POST Возвращает строку с новым значением, сохраняя префикс и кавычки. @@ -321,6 +333,7 @@ def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_ app_logger.error("[_update_yaml_file][Failure] Error performing raw replacement in %s: %s", file_path, e) # #endregion _update_yaml_file # #region create_dashboard_export [TYPE Function] +# @ingroup Core # @BRIEF Создает ZIP-архив из указанных исходных путей. # @PRE source_paths должен содержать существующие пути. # @POST ZIP-архив создан по пути zip_path. @@ -344,6 +357,7 @@ def create_dashboard_export(zip_path: str | Path, source_paths: list[str | Path] return False # #endregion create_dashboard_export # #region sanitize_filename [TYPE Function] +# @ingroup Core # @BRIEF Очищает строку от символов, недопустимых в именах файлов. # @PRE filename должен быть строкой. # @POST Возвращает строку без спецсимволов. @@ -352,6 +366,7 @@ def sanitize_filename(filename: str) -> str: return re.sub(r'[\\/*?:"<>|]', "_", filename).strip() # #endregion sanitize_filename # #region get_filename_from_headers [TYPE Function] +# @ingroup Core # @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'. # @PRE headers должен быть словарем заголовков. # @POST Возвращает имя файла или None, если заголовок отсутствует. @@ -363,6 +378,7 @@ def get_filename_from_headers(headers: dict) -> str | None: return None # #endregion get_filename_from_headers # #region consolidate_archive_folders [TYPE Function] +# @ingroup Core # @BRIEF Консолидирует директории архивов на основе общего слага в имени. # @PRE root_directory должен быть объектом Path к существующей директории. # @POST Директории с одинаковым префиксом объединены в одну. @@ -416,6 +432,7 @@ def consolidate_archive_folders(root_directory: Path) -> None: app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e) # #endregion consolidate_archive_folders # #region AsyncFileIOWrappers [C:3] [TYPE Module] [SEMANTICS async, fileio, aiofiles, run_blocking] +# @defgroup Core Module group. # @BRIEF Async wrappers for file I/O operations using run_blocking executor and aiofiles. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [BlockingExecutorsModule] @@ -428,6 +445,7 @@ from .executors import run_blocking # noqa: E402 # #region aread_dashboard_from_disk [C:3] [TYPE Function] [SEMANTICS async, file, read] +# @ingroup Core # @BRIEF Async read dashboard file from disk using aiofiles for large files or run_blocking fallback. # @PRE file_path must point to an existing file. # @POST Returns file bytes content. @@ -451,6 +469,7 @@ async def aread_dashboard_from_disk(file_path: str) -> bytes: # #region asave_and_unpack_dashboard [C:3] [TYPE Function] [SEMANTICS async, zip, save, unpack] +# @ingroup Core # @BRIEF Async save ZIP content to disk and optionally unpack it. # @PRE zip_content must be valid ZIP archive bytes. # @POST ZIP file saved to target_dir; if unpack=True, contents extracted. @@ -499,6 +518,7 @@ def _unzip_to_directory(zip_path: str, output_path: str) -> None: # #region acleanup_directory [C:2] [TYPE Function] [SEMANTICS async, cleanup, directory] +# @ingroup Core # @BRIEF Async remove directory tree using run_blocking executor. # @PRE path is a valid filesystem path. # @POST Directory and all contents removed. @@ -510,6 +530,7 @@ async def acleanup_directory(path: str) -> None: # #region aread_bytes [C:2] [TYPE Function] [SEMANTICS async, read, bytes] +# @ingroup Core # @BRIEF Async read file bytes from a Path using run_blocking executor. # @PRE path must point to an existing file. # @POST Returns file bytes content. diff --git a/backend/src/core/utils/matching.py b/backend/src/core/utils/matching.py index 04d2782b..9e8a93a3 100644 --- a/backend/src/core/utils/matching.py +++ b/backend/src/core/utils/matching.py @@ -1,4 +1,5 @@ # #region FuzzyMatching [TYPE Module] [SEMANTICS fuzzy, matching, rapidfuzz, database, mapping] +# @defgroup Core Module group. # # @BRIEF Provides utility functions for fuzzy matching database names. # @LAYER Core @@ -11,6 +12,7 @@ from rapidfuzz import fuzz, process # #region suggest_mappings [TYPE Function] +# @ingroup Core # @BRIEF Suggests mappings between source and target databases using fuzzy matching. # @PRE source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'. # @POST Returns a list of suggested mappings with confidence scores. diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index a00a812e..5d427891 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -1,4 +1,5 @@ # #region NetworkModule [C:3] [TYPE Module] [SEMANTICS network, http, retry, tenacity, session] +# @defgroup Core Module group. # # @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок. # @LAYER Infrastructure @@ -45,6 +46,7 @@ class AuthenticationError(SupersetAPIError): # #endregion __init__ # #endregion AuthenticationError # #region PermissionDeniedError [TYPE Class] +# @defgroup Core Module group. # @BRIEF Exception raised when access is denied. class PermissionDeniedError(AuthenticationError): # #region __init__ [TYPE Function] @@ -57,6 +59,7 @@ class PermissionDeniedError(AuthenticationError): # #endregion __init__ # #endregion PermissionDeniedError # #region DashboardNotFoundError [TYPE Class] +# @defgroup Core Module group. # @BRIEF Exception raised when a dashboard cannot be found. class DashboardNotFoundError(SupersetAPIError): # #region __init__ [TYPE Function] @@ -69,9 +72,11 @@ class DashboardNotFoundError(SupersetAPIError): # #endregion __init__ # #endregion DashboardNotFoundError # #region NetworkError [TYPE Class] +# @defgroup Core Module group. # @BRIEF Exception raised when a network level error occurs. class NetworkError(Exception): # #region NetworkError.__init__ [TYPE Function] +# @ingroup Core # @PURPOSE: Initializes the network error. # @PRE message is a string. # @POST NetworkError is initialized. @@ -82,6 +87,7 @@ class NetworkError(Exception): # #endregion NetworkError.__init__ # #endregion NetworkError # #region SupersetAuthCache [TYPE Class] +# @defgroup Core Module group. # @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials. # @PRE base_url and username are stable strings. # @POST Cached entries expire automatically by TTL and can be reused across requests. @@ -97,6 +103,7 @@ class SupersetAuthCache: return (str(base_url or "").strip(), username, bool(verify_ssl)) @classmethod # #region SupersetAuthCache.get [TYPE Function] +# @ingroup Core def get(cls, key: tuple[str, str, bool]) -> dict[str, str] | None: now = time.time() with cls._lock: @@ -118,6 +125,7 @@ class SupersetAuthCache: # #endregion SupersetAuthCache.get @classmethod # #region SupersetAuthCache.set [TYPE Function] +# @ingroup Core def set(cls, key: tuple[str, str, bool], tokens: dict[str, str], ttl_seconds: int | None = None) -> None: normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1) with cls._lock: diff --git a/backend/src/core/utils/superset_compilation_adapter.py b/backend/src/core/utils/superset_compilation_adapter.py index 133791cb..14d9edf1 100644 --- a/backend/src/core/utils/superset_compilation_adapter.py +++ b/backend/src/core/utils/superset_compilation_adapter.py @@ -1,4 +1,5 @@ # #region SupersetCompilationAdapter [C:4] [TYPE Module] [SEMANTICS superset, preview, compile, execution, adapter] +# @defgroup Core Module group. # @BRIEF Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context. # @LAYER Infrastructure # @RELATION CALLS -> [SupersetClient] @@ -10,6 +11,7 @@ from __future__ import annotations # #region SupersetCompilationAdapter.imports [TYPE Block] +# @ingroup Core from dataclasses import dataclass from datetime import UTC, datetime from typing import Any @@ -22,6 +24,7 @@ from src.models.dataset_review import CompiledPreview, PreviewStatus # #endregion SupersetCompilationAdapter.imports # #region PreviewCompilationPayload [C:2] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Typed preview payload for Superset-side compilation. @dataclass(frozen=True) class PreviewCompilationPayload: @@ -32,6 +35,7 @@ class PreviewCompilationPayload: effective_filters: list[dict[str, Any]] # #endregion PreviewCompilationPayload # #region SqlLabLaunchPayload [C:2] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Typed SQL Lab payload for audited launch handoff. @dataclass(frozen=True) class SqlLabLaunchPayload: @@ -42,6 +46,7 @@ class SqlLabLaunchPayload: template_params: dict[str, Any] # #endregion SqlLabLaunchPayload # #region SupersetCompilationAdapter [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication. # @RELATION CALLS -> [SupersetClient] # @PRE environment is configured and Superset is reachable for the target session. @@ -49,6 +54,7 @@ class SqlLabLaunchPayload: # @SIDE_EFFECT issues network requests to Superset API surfaces. class SupersetCompilationAdapter: # #region SupersetCompilationAdapter.__init__ [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Bind adapter to one Superset environment and client instance. def __init__( self, environment: Environment, client: SupersetClient | None = None @@ -57,6 +63,7 @@ class SupersetCompilationAdapter: self.client = client or SupersetClient(environment) # #endregion SupersetCompilationAdapter.__init__ # #region SupersetCompilationAdapter._supports_client_method [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Detect explicitly implemented client capabilities without treating loose mocks as real methods. def _supports_client_method(self, method_name: str) -> bool: client_dict = getattr(self.client, "__dict__", {}) @@ -65,6 +72,7 @@ class SupersetCompilationAdapter: return callable(getattr(type(self.client), method_name, None)) # #endregion SupersetCompilationAdapter._supports_client_method # #region SupersetCompilationAdapter.compile_preview [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs. # @RELATION CALLS -> [SupersetCompilationAdapter._request_superset_preview] # @PRE dataset_id and effective inputs are available for the current session. @@ -153,6 +161,7 @@ class SupersetCompilationAdapter: return preview # #endregion SupersetCompilationAdapter.compile_preview # #region SupersetCompilationAdapter.mark_preview_stale [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Invalidate previous preview after mapping or value changes. # @PRE preview is a persisted preview artifact or current in-memory snapshot. # @POST preview status becomes stale without fabricating a replacement artifact. @@ -161,6 +170,7 @@ class SupersetCompilationAdapter: return preview # #endregion SupersetCompilationAdapter.mark_preview_stale # #region SupersetCompilationAdapter.create_sql_lab_session [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Create the canonical audited execution session after all launch gates pass. # @RELATION CALLS -> [SupersetCompilationAdapter._request_sql_lab_session] # @PRE compiled_sql is Superset-originated and launch gates are already satisfied. @@ -217,6 +227,7 @@ class SupersetCompilationAdapter: return sql_lab_session_ref # #endregion SupersetCompilationAdapter.create_sql_lab_session # #region SupersetCompilationAdapter._request_superset_preview [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Request preview compilation through explicit client support backed by real Superset endpoints only. # @RELATION CALLS -> [EXT:method:SupersetClient.compile_dataset_preview] # @PRE payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt. @@ -355,6 +366,7 @@ class SupersetCompilationAdapter: raise RuntimeError(str(exc)) from exc # #endregion SupersetCompilationAdapter._request_superset_preview # #region SupersetCompilationAdapter._request_sql_lab_session [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response. # @RELATION CALLS -> [EXT:method:SupersetClient.get_dataset] # @PRE payload carries non-empty Superset-originated SQL and a preview identifier for the current launch. @@ -410,6 +422,7 @@ class SupersetCompilationAdapter: ) # #endregion SupersetCompilationAdapter._request_sql_lab_session # #region SupersetCompilationAdapter._normalize_preview_response [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure. # @RELATION DEPENDS_ON -> [CompiledPreview] def _normalize_preview_response(self, response: Any) -> dict[str, Any] | None: diff --git a/backend/src/core/utils/superset_context_extractor/__init__.py b/backend/src/core/utils/superset_context_extractor/__init__.py index 79b456dd..b70b231d 100644 --- a/backend/src/core/utils/superset_context_extractor/__init__.py +++ b/backend/src/core/utils/superset_context_extractor/__init__.py @@ -1,4 +1,5 @@ # #region SupersetContextExtractorPackage [C:4] [TYPE Module] [SEMANTICS superset, package, dashboard, dataset] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers. # @RELATION DEPENDS_ON -> [ImportedFilter] @@ -25,6 +26,7 @@ from ._templates import SupersetContextTemplatesMixin # #region SupersetContextExtractor [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. # @RELATION DEPENDS_ON -> [Environment] # @RELATION INHERITS -> [SupersetContextExtractorBase] diff --git a/backend/src/core/utils/superset_context_extractor/_base.py b/backend/src/core/utils/superset_context_extractor/_base.py index 8213bc70..29f91d59 100644 --- a/backend/src/core/utils/superset_context_extractor/_base.py +++ b/backend/src/core/utils/superset_context_extractor/_base.py @@ -1,4 +1,5 @@ # #region SupersetContextExtractorBase [C:4] [TYPE Module] [SEMANTICS superset, context, extract, base, recovery] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers. # @RELATION DEPENDS_ON -> [ImportedFilter] @@ -24,6 +25,7 @@ from ...superset_client import SupersetClient # #endregion _base_imports logger = cast(Any, logger) # #region SupersetParsedContext [C:2] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Normalized output of Superset link parsing for session intake and recovery. @dataclass class SupersetParsedContext: @@ -40,6 +42,7 @@ class SupersetParsedContext: dataset_payload: dict[str, Any] | None = None # #endregion SupersetParsedContext # #region SupersetContextExtractorBase [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers. # @RELATION DEPENDS_ON -> [Environment] # @PRE constructor receives a configured environment with a usable Superset base URL. @@ -47,6 +50,7 @@ class SupersetParsedContext: # @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient. class SupersetContextExtractorBase: # #region SupersetContextExtractorBase.__init__ [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Bind extractor to one Superset environment and client instance. def __init__( self, environment: Environment, client: SupersetClient | None = None @@ -55,6 +59,7 @@ class SupersetContextExtractorBase: self.client = client or SupersetClient(environment) # #endregion SupersetContextExtractorBase.__init__ # #region SupersetContextExtractorBase.build_recovery_summary [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX. def build_recovery_summary( self, parsed_context: SupersetParsedContext @@ -70,6 +75,7 @@ class SupersetContextExtractorBase: } # #endregion SupersetContextExtractorBase.build_recovery_summary # #region SupersetContextExtractorBase._extract_numeric_identifier [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path. def _extract_numeric_identifier( self, path_parts: list[str], resource_name: str @@ -88,6 +94,7 @@ class SupersetContextExtractorBase: return int(candidate) # #endregion SupersetContextExtractorBase._extract_numeric_identifier # #region SupersetContextExtractorBase._extract_dashboard_reference [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path. def _extract_dashboard_reference(self, path_parts: list[str]) -> str | None: if "dashboard" not in path_parts: @@ -104,6 +111,7 @@ class SupersetContextExtractorBase: return candidate # #endregion SupersetContextExtractorBase._extract_dashboard_reference # #region SupersetContextExtractorBase._extract_dashboard_permalink_key [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Extract a dashboard permalink key from a Superset URL path. def _extract_dashboard_permalink_key(self, path_parts: list[str]) -> str | None: if "dashboard" not in path_parts: @@ -121,6 +129,7 @@ class SupersetContextExtractorBase: return permalink_key # #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key # #region SupersetContextExtractorBase._extract_dashboard_id_from_state [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Extract a dashboard identifier from returned permalink state when present. def _extract_dashboard_id_from_state(self, state: dict[str, Any]) -> int | None: return self._search_nested_numeric_key( @@ -129,6 +138,7 @@ class SupersetContextExtractorBase: ) # #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state # #region SupersetContextExtractorBase._extract_chart_id_from_state [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent. def _extract_chart_id_from_state(self, state: dict[str, Any]) -> int | None: return self._search_nested_numeric_key( @@ -137,6 +147,7 @@ class SupersetContextExtractorBase: ) # #endregion SupersetContextExtractorBase._extract_chart_id_from_state # #region SupersetContextExtractorBase._search_nested_numeric_key [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set. # @RELATION DEPENDS_ON -> [SupersetContextExtractorBase] def _search_nested_numeric_key( @@ -161,6 +172,7 @@ class SupersetContextExtractorBase: return None # #endregion SupersetContextExtractorBase._search_nested_numeric_key # #region SupersetContextExtractorBase._decode_query_state [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Decode query-string structures used by Superset URL state transport. def _decode_query_state(self, query_params: dict[str, list[str]]) -> dict[str, Any]: query_state: dict[str, Any] = {} diff --git a/backend/src/core/utils/superset_context_extractor/_filters.py b/backend/src/core/utils/superset_context_extractor/_filters.py index eb456e67..0f488dfa 100644 --- a/backend/src/core/utils/superset_context_extractor/_filters.py +++ b/backend/src/core/utils/superset_context_extractor/_filters.py @@ -1,4 +1,5 @@ # #region SupersetContextFiltersExtractMixin [C:3] [TYPE Module] [SEMANTICS superset, filter, native, context, extract] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data). # @RELATION DEPENDS_ON -> [SupersetContextExtractorBase] @@ -14,9 +15,11 @@ from ...logger import logger as app_logger app_logger = cast(Any, app_logger) # #endregion _filters_imports # #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing query-state filter extraction for the composed SupersetContextExtractor. class SupersetContextFiltersExtractMixin: # #region SupersetContextFiltersExtractMixin._extract_imported_filters [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values. def _extract_imported_filters( self, query_state: dict[str, Any] diff --git a/backend/src/core/utils/superset_context_extractor/_parsing.py b/backend/src/core/utils/superset_context_extractor/_parsing.py index 5cfa50e0..a6f50515 100644 --- a/backend/src/core/utils/superset_context_extractor/_parsing.py +++ b/backend/src/core/utils/superset_context_extractor/_parsing.py @@ -1,4 +1,5 @@ # #region SupersetContextParsingMixin [C:4] [TYPE Module] [SEMANTICS superset, transform, dashboard, dataset, session, review] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. # @RELATION CALLS -> [SupersetClient] @@ -16,9 +17,11 @@ from ._base import SupersetParsedContext logger = cast(Any, logger) # #endregion _parsing_imports # #region SupersetContextParsingMixin [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor. class SupersetContextParsingMixin: # #region SupersetContextParsingMixin.parse_superset_link [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs. # @RELATION CALLS -> [EXT:method:SupersetClient.get_dashboard_detail] # @PRE link is a non-empty Superset URL compatible with the configured environment. @@ -315,6 +318,7 @@ class SupersetContextParsingMixin: return result # #endregion SupersetContextParsingMixin.parse_superset_link # #region SupersetContextParsingMixin._recover_dataset_binding_from_dashboard [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers. # @RELATION CALLS -> [EXT:method:SupersetClient.get_dashboard_detail] async def _recover_dataset_binding_from_dashboard( diff --git a/backend/src/core/utils/superset_context_extractor/_pii.py b/backend/src/core/utils/superset_context_extractor/_pii.py index 88e87fe6..60e15223 100644 --- a/backend/src/core/utils/superset_context_extractor/_pii.py +++ b/backend/src/core/utils/superset_context_extractor/_pii.py @@ -1,4 +1,5 @@ # #region SupersetContextExtractorPII [C:3] [TYPE Module] [SEMANTICS superset, dataset, assistant, review] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers. # #endregion SupersetContextExtractorPII @@ -23,6 +24,7 @@ _MIXED_IDENTIFIER_PATTERN = re.compile( # #region mask_pii_value [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context. def mask_pii_value(value: Any) -> tuple[Any, bool]: if isinstance(value, str): @@ -50,6 +52,7 @@ def mask_pii_value(value: Any) -> tuple[Any, bool]: # #region sanitize_imported_filter_for_assistant [C:2] [TYPE Function] +# @ingroup Core # @BRIEF Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected. def sanitize_imported_filter_for_assistant( filter_payload: dict[str, Any], diff --git a/backend/src/core/utils/superset_context_extractor/_recovery.py b/backend/src/core/utils/superset_context_extractor/_recovery.py index a1373bb9..4477a262 100644 --- a/backend/src/core/utils/superset_context_extractor/_recovery.py +++ b/backend/src/core/utils/superset_context_extractor/_recovery.py @@ -1,4 +1,5 @@ # #region SupersetContextRecoveryMixin [C:4] [TYPE Module] [SEMANTICS superset, context, recovery, dashboard, dataset] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Recover imported filters from Superset parsed context and dashboard metadata. # @RELATION CALLS -> [SupersetClient] @@ -17,9 +18,11 @@ from ._base import SupersetParsedContext logger = cast(Any, logger) # #endregion _recovery_imports # #region SupersetContextRecoveryMixin [C:4] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing filter recovery for the composed SupersetContextExtractor. class SupersetContextRecoveryMixin: # #region SupersetContextRecoveryMixin.recover_imported_filters [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Build imported filter entries from URL state and Superset-side saved context. # @RELATION CALLS -> [EXT:method:SupersetClient.get_dashboard] # @PRE parsed_context comes from a successful Superset link parse for one environment. @@ -242,6 +245,7 @@ class SupersetContextRecoveryMixin: return recovered_filters # #endregion SupersetContextRecoveryMixin.recover_imported_filters # #region SupersetContextRecoveryMixin._normalize_imported_filter_payload [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state. def _normalize_imported_filter_payload( self, diff --git a/backend/src/core/utils/superset_context_extractor/_templates.py b/backend/src/core/utils/superset_context_extractor/_templates.py index e824b7c3..3f2b6e17 100644 --- a/backend/src/core/utils/superset_context_extractor/_templates.py +++ b/backend/src/core/utils/superset_context_extractor/_templates.py @@ -1,4 +1,5 @@ # #region SupersetContextTemplatesMixin [C:3] [TYPE Module] [SEMANTICS superset, search, dataset, execution] +# @defgroup Core Module group. # @LAYER Infrastructure # @BRIEF Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution. # @RELATION DEPENDS_ON -> [TemplateVariable] @@ -15,9 +16,11 @@ from ...logger import belief_scope, logger logger = cast(Any, logger) # #endregion _templates_imports # #region SupersetContextTemplatesMixin [C:3] [TYPE Class] +# @defgroup Core Module group. # @BRIEF Mixin providing template variable discovery for the composed SupersetContextExtractor. class SupersetContextTemplatesMixin: # #region SupersetContextTemplatesMixin.discover_template_variables [TYPE Function] [C:4] +# @ingroup Core # @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields. # @PRE dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available. # @POST returns deduplicated explicit variable records without executing Jinja or fabricating runtime values. @@ -111,6 +114,7 @@ class SupersetContextTemplatesMixin: return discovered # #endregion SupersetContextTemplatesMixin.discover_template_variables # #region SupersetContextTemplatesMixin._collect_query_bearing_expressions [TYPE Function] [C:3] +# @ingroup Core # @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. # @RELATION DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables] def _collect_query_bearing_expressions( @@ -147,6 +151,7 @@ class SupersetContextTemplatesMixin: return expressions # #endregion SupersetContextTemplatesMixin._collect_query_bearing_expressions # #region SupersetContextTemplatesMixin._append_template_variable [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Append one deduplicated template-variable descriptor. def _append_template_variable( self, @@ -177,6 +182,7 @@ class SupersetContextTemplatesMixin: ) # #endregion SupersetContextTemplatesMixin._append_template_variable # #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it. def _extract_primary_jinja_identifier(self, expression: str) -> str | None: matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip()) @@ -197,6 +203,7 @@ class SupersetContextTemplatesMixin: return candidate # #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier # #region SupersetContextTemplatesMixin._normalize_default_literal [TYPE Function] [C:2] +# @ingroup Core # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values. def _normalize_default_literal(self, literal: str | None) -> Any: normalized_literal = str(literal or "").strip() diff --git a/backend/src/core/ws_log_handler.py b/backend/src/core/ws_log_handler.py index 6eb73968..2dcdc7ff 100644 --- a/backend/src/core/ws_log_handler.py +++ b/backend/src/core/ws_log_handler.py @@ -1,4 +1,5 @@ # #region WsLogHandlerModule [C:3] [TYPE Module] [SEMANTICS pydantic, dto, log-entry] +# @defgroup Core Module group. # @BRIEF WebSocket log handler module providing LogEntry DTO and WebSocketLogHandler for buffered log streaming. # @RELATION DEPENDS_ON -> [LogEntry] # @RELATION DEPENDS_ON -> [CotJsonFormatter] @@ -20,6 +21,7 @@ class LogEntry(BaseModel): # #region WebSocketLogHandler [C:3] [TYPE Class] [SEMANTICS logging,handler,websocket,buffer] +# @defgroup Core Module group. # @BRIEF Custom logging handler that captures log records into a fixed-capacity buffer for WebSocket streaming. # @RELATION DEPENDS_ON -> [LogEntry] # @RELATION DEPENDS_ON -> [CotJsonFormatter] @@ -36,6 +38,7 @@ class WebSocketLogHandler(logging.Handler): # #endregion WebSocketLogHandler.__init__ # #region WebSocketLogHandler.emit [C:2] [TYPE Function] [SEMANTICS log,capture,format,buffer] +# @ingroup Core # @BRIEF Captures a log record, formats it, and stores it in the buffer as a LogEntry. def emit(self, record: logging.LogRecord): try: @@ -57,6 +60,7 @@ class WebSocketLogHandler(logging.Handler): # #endregion WebSocketLogHandler.emit # #region WebSocketLogHandler.get_recent_logs [C:2] [TYPE Function] [SEMANTICS log,retrieve,buffer] +# @ingroup Core # @BRIEF Returns a list of recent log entries from the buffer. def get_recent_logs(self) -> list[LogEntry]: """ diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index 24eb713c..b0877d80 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -1,4 +1,5 @@ # #region AppDependencies [C:4] [TYPE Module] [SEMANTICS fastapi, schedule, auth, api_key] +# @defgroup Module Module group. # @BRIEF Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports. # @LAYER Core # @PURPOSE Provides shared instances to app and routers. @@ -218,6 +219,7 @@ class APIKeyPrincipal: # #region require_api_key_or_jwt [C:4] [TYPE Function] +# @ingroup Module # @BRIEF Factory: creates a dependency that accepts either a valid API key (with permission check) # OR a JWT (with standard has_permission check). Returns the authenticated user/principal name. # JWT takes precedence when both auth methods are present. @@ -414,6 +416,7 @@ async def check_api_key_environment_scope(request: Request, db, target_env_id: s # #region get_api_key_principal [C:3] [TYPE Function] +# @ingroup Module # @BRIEF FastAPI dependency that reads X-API-Key header and returns APIKeyPrincipal or None. # @PRE X-API-Key header may be present in the request. # @POST If valid key: returns APIKeyPrincipal; if no header: returns None; if invalid: raises 401. @@ -497,6 +500,7 @@ oauth2_scheme_optional = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_e # #region get_current_user [C:3] [TYPE Function] +# @ingroup Module # @RELATION CALLS -> AuthRepository # @RELATION CALLS -> [is_token_blacklisted] # @BRIEF Dependency for retrieving currently authenticated user from a JWT. @@ -536,6 +540,7 @@ def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db # #region has_permission [C:3] [TYPE Function] +# @ingroup Module # @RELATION CALLS -> AuthRepository # @BRIEF Dependency for checking if the current user has a specific permission. # @PRE User is authenticated. diff --git a/backend/src/models/__init__.py b/backend/src/models/__init__.py index a1db4503..30ea8d07 100644 --- a/backend/src/models/__init__.py +++ b/backend/src/models/__init__.py @@ -1,3 +1,4 @@ # #region ModelsPackage [TYPE Package] [SEMANTICS model, package, init] +# @ingroup Models # @BRIEF Domain model package root. # #endregion ModelsPackage diff --git a/backend/src/models/assistant.py b/backend/src/models/assistant.py index 053332fc..c1051424 100644 --- a/backend/src/models/assistant.py +++ b/backend/src/models/assistant.py @@ -1,4 +1,5 @@ # #region AssistantModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, assistant, model, schema, audit, assistant-audit-record] +# @defgroup Models Module group. # @BRIEF SQLAlchemy models for assistant audit trail and confirmation tokens. # @LAYER Domain # @RELATION DEPENDS_ON -> MappingModels @@ -14,6 +15,7 @@ from .mapping import Base # #region AssistantAuditRecord [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Store audit decisions and outcomes produced by assistant command handling. # @RELATION INHERITS -> MappingModels # @PRE user_id must identify the actor for every record. @@ -35,6 +37,7 @@ class AssistantAuditRecord(Base): # #region AssistantMessageRecord [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Persist chat history entries for assistant conversations. # @RELATION INHERITS -> MappingModels # @PRE user_id, conversation_id, role and text must be present. @@ -58,6 +61,7 @@ class AssistantMessageRecord(Base): # #region AssistantConfirmationRecord [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Persist risky operation confirmation tokens with lifecycle state. # @RELATION INHERITS -> MappingModels # @PRE intent/dispatch and expiry timestamp must be provided. diff --git a/backend/src/models/auth.py b/backend/src/models/auth.py index cfda6fd2..226c6278 100644 --- a/backend/src/models/auth.py +++ b/backend/src/models/auth.py @@ -1,4 +1,5 @@ # #region AuthModels [C:5] [TYPE Module] [SEMANTICS sqlalchemy, auth, model, schema, user] +# @defgroup Models Module group. # @BRIEF SQLAlchemy models for multi-user authentication and authorization. # @LAYER Domain # @RELATION INHERITS -> [EXT:Library:SQLAlchemy.Base] @@ -19,6 +20,7 @@ from .mapping import Base # #region generate_uuid [TYPE Function] +# @ingroup Models # @BRIEF Generates a unique UUID string. # @POST Returns a string representation of a new UUID. # @RELATION DEPENDS_ON -> [EXT:Python:uuid] @@ -29,6 +31,7 @@ def generate_uuid(): # #endregion generate_uuid # #region user_roles [TYPE Table] +# @ingroup Models # @BRIEF Association table for many-to-many relationship between Users and Roles. # @RELATION DEPENDS_ON -> [EXT:Library:SQLAlchemy.Base] # @RELATION DEPENDS_ON -> [User] @@ -42,6 +45,7 @@ user_roles = Table( # #endregion user_roles # #region role_permissions [TYPE Table] +# @ingroup Models # @BRIEF Association table for many-to-many relationship between Roles and Permissions. # @RELATION DEPENDS_ON -> [EXT:Library:SQLAlchemy.Base] # @RELATION DEPENDS_ON -> [Role] @@ -56,6 +60,7 @@ role_permissions = Table( # #region User [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents an identity that can authenticate to the system. # @RELATION BINDS_TO -> [Role] class User(Base): @@ -79,6 +84,7 @@ class User(Base): # #region Role [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a collection of permissions. # @RELATION BINDS_TO -> [User] # @RELATION BINDS_TO -> [Permission] @@ -100,6 +106,7 @@ class Role(Base): # #region Permission [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a specific capability within the system. # @RELATION BINDS_TO -> [Role] class Permission(Base): @@ -118,6 +125,7 @@ class Permission(Base): # #region TokenBlacklist [TYPE Class] +# @defgroup Models Module group. # @BRIEF Stores SHA-256 hashes of revoked JWTs for server-side token blacklisting. # @INVARIANT Only the SHA-256 hash of the token is stored — never the raw token. # Expired entries are cleaned up by _prune_blacklist(). @@ -134,6 +142,7 @@ class TokenBlacklist(Base): # #endregion TokenBlacklist # #region ADGroupMapping [TYPE Class] +# @defgroup Models Module group. # @BRIEF Maps an Active Directory group to a local System Role. # @RELATION DEPENDS_ON -> [Role] class ADGroupMapping(Base): diff --git a/backend/src/models/clean_release.py b/backend/src/models/clean_release.py index 31672bcf..30b5012f 100644 --- a/backend/src/models/clean_release.py +++ b/backend/src/models/clean_release.py @@ -1,4 +1,5 @@ # #region CleanReleaseModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, clean-release, model, schema, schedule, release] +# @defgroup Models Module group. # @BRIEF Define canonical clean release domain entities and lifecycle guards. # @LAYER Domain # @RELATION DEPENDS_ON -> MappingModels @@ -22,6 +23,7 @@ from .mapping import Base # #region ExecutionMode [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible execution mode enum for legacy TUI/orchestrator tests. class ExecutionMode(str, Enum): TUI = "TUI" @@ -30,6 +32,7 @@ class ExecutionMode(str, Enum): # #endregion ExecutionMode # #region CheckFinalStatus [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible final status enum for legacy TUI/orchestrator tests. class CheckFinalStatus(str, Enum): COMPLIANT = "COMPLIANT" @@ -39,6 +42,7 @@ class CheckFinalStatus(str, Enum): # #endregion CheckFinalStatus # #region CheckStageName [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible stage name enum for legacy TUI/orchestrator tests. class CheckStageName(str, Enum): DATA_PURITY = "DATA_PURITY" @@ -48,6 +52,7 @@ class CheckStageName(str, Enum): # #endregion CheckStageName # #region CheckStageStatus [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible stage status enum for legacy TUI/orchestrator tests. class CheckStageStatus(str, Enum): PASS = "PASS" @@ -57,6 +62,7 @@ class CheckStageStatus(str, Enum): # #endregion CheckStageStatus # #region CheckStageResult [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible stage result container for legacy TUI/orchestrator tests. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class CheckStageResult: @@ -66,12 +72,14 @@ class CheckStageResult: # #endregion CheckStageResult # #region ProfileType [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible profile enum for legacy TUI bootstrap logic. class ProfileType(str, Enum): ENTERPRISE_CLEAN = "enterprise-clean" # #endregion ProfileType # #region RegistryStatus [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible registry status enum for legacy TUI bootstrap logic. class RegistryStatus(str, Enum): ACTIVE = "ACTIVE" @@ -79,6 +87,7 @@ class RegistryStatus(str, Enum): # #endregion RegistryStatus # #region ReleaseCandidateStatus [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible release candidate status enum for legacy TUI. class ReleaseCandidateStatus(str, Enum): DRAFT = CandidateStatus.DRAFT.value @@ -96,6 +105,7 @@ class ReleaseCandidateStatus(str, Enum): # #endregion ReleaseCandidateStatus # #region ResourceSourceEntry [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible source entry model for legacy TUI bootstrap logic. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class ResourceSourceEntry: @@ -107,6 +117,7 @@ class ResourceSourceEntry: # #endregion ResourceSourceEntry # #region ResourceSourceRegistry [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible source registry model for legacy TUI bootstrap logic. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class ResourceSourceRegistry: @@ -138,6 +149,7 @@ class ResourceSourceRegistry: # #endregion ResourceSourceRegistry # #region CleanProfilePolicy [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible policy model for legacy TUI bootstrap logic. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class CleanProfilePolicy: @@ -179,6 +191,7 @@ class CleanProfilePolicy: # #endregion CleanProfilePolicy # #region ComplianceCheckRun [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible run model for legacy TUI typing/import compatibility. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class ComplianceCheckRun: @@ -226,6 +239,7 @@ class ComplianceCheckRun: # #endregion ComplianceCheckRun # #region ReleaseCandidate [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents the release unit being prepared and governed. # @PRE id, version, source_snapshot_ref are non-empty. # @POST status advances only through legal transitions. @@ -288,6 +302,7 @@ class ReleaseCandidate(Base): # #endregion ReleaseCandidate # #region CandidateArtifact [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents one artifact associated with a release candidate. class CandidateArtifact(Base): __tablename__ = "clean_release_artifacts" @@ -305,6 +320,7 @@ class CandidateArtifact(Base): # #endregion CandidateArtifact # #region ManifestItem [TYPE Class] +# @defgroup Models Module group. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class ManifestItem: path: str @@ -315,6 +331,7 @@ class ManifestItem: # #endregion ManifestItem # #region ManifestSummary [TYPE Class] +# @defgroup Models Module group. @pydantic_dataclass(config=ConfigDict(validate_assignment=True)) class ManifestSummary: included_count: int @@ -323,6 +340,7 @@ class ManifestSummary: # #endregion ManifestSummary # #region DistributionManifest [TYPE Class] +# @defgroup Models Module group. # @BRIEF Immutable snapshot of the candidate payload. # @INVARIANT Immutable after creation. class DistributionManifest(Base): @@ -411,6 +429,7 @@ class DistributionManifest(Base): # #endregion DistributionManifest # #region SourceRegistrySnapshot [TYPE Class] +# @defgroup Models Module group. # @BRIEF Immutable registry snapshot for allowed sources. class SourceRegistrySnapshot(Base): __tablename__ = "clean_release_registry_snapshots" @@ -426,6 +445,7 @@ class SourceRegistrySnapshot(Base): # #endregion SourceRegistrySnapshot # #region CleanPolicySnapshot [TYPE Class] +# @defgroup Models Module group. # @BRIEF Immutable policy snapshot used to evaluate a run. class CleanPolicySnapshot(Base): __tablename__ = "clean_release_policy_snapshots" @@ -440,6 +460,7 @@ class CleanPolicySnapshot(Base): # #endregion CleanPolicySnapshot # #region ComplianceRun [TYPE Class] +# @defgroup Models Module group. # @BRIEF Operational record for one compliance execution. class ComplianceRun(Base): __tablename__ = "clean_release_compliance_runs" @@ -465,6 +486,7 @@ class ComplianceRun(Base): # #endregion ComplianceRun # #region ComplianceStageRun [TYPE Class] +# @defgroup Models Module group. # @BRIEF Stage-level execution record inside a run. class ComplianceStageRun(Base): __tablename__ = "clean_release_compliance_stage_runs" @@ -480,6 +502,7 @@ class ComplianceStageRun(Base): # #endregion ComplianceStageRun # #region ViolationSeverity [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible violation severity enum for legacy clean-release tests. class ViolationSeverity(str, Enum): CRITICAL = "CRITICAL" @@ -488,6 +511,7 @@ class ViolationSeverity(str, Enum): # #endregion ViolationSeverity # #region ViolationCategory [TYPE Class] +# @defgroup Models Module group. # @BRIEF Backward-compatible violation category enum for legacy clean-release tests. class ViolationCategory(str, Enum): DATA_PURITY = "DATA_PURITY" @@ -498,6 +522,7 @@ class ViolationCategory(str, Enum): # #endregion ViolationCategory # #region ComplianceViolation [TYPE Class] +# @defgroup Models Module group. # @BRIEF Violation produced by a stage. class ComplianceViolation(Base): __tablename__ = "clean_release_compliance_violations" @@ -574,6 +599,7 @@ class ComplianceViolation(Base): # #endregion ComplianceViolation # #region ComplianceReport [TYPE Class] +# @defgroup Models Module group. # @BRIEF Immutable result derived from a completed run. # @INVARIANT Immutable after creation. class ComplianceReport(Base): @@ -648,6 +674,7 @@ class ComplianceReport(Base): # #endregion ComplianceReport # #region ApprovalDecision [TYPE Class] +# @defgroup Models Module group. # @BRIEF Approval or rejection bound to a candidate and report. class ApprovalDecision(Base): __tablename__ = "clean_release_approval_decisions" @@ -662,6 +689,7 @@ class ApprovalDecision(Base): # #endregion ApprovalDecision # #region PublicationRecord [TYPE Class] +# @defgroup Models Module group. # @BRIEF Publication or revocation record. class PublicationRecord(Base): __tablename__ = "clean_release_publication_records" @@ -677,6 +705,7 @@ class PublicationRecord(Base): # #endregion PublicationRecord # #region CleanReleaseAuditLog [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a persistent audit log entry for clean release actions. import uuid diff --git a/backend/src/models/config.py b/backend/src/models/config.py index 88648b90..0034c181 100644 --- a/backend/src/models/config.py +++ b/backend/src/models/config.py @@ -1,4 +1,5 @@ # #region ConfigModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, model, schema, notification, app-config-record] +# @defgroup Models Module group. # # @BRIEF Defines SQLAlchemy persistence models for application and notification configuration records. # @LAYER Domain @@ -13,6 +14,7 @@ from .mapping import Base # #region AppConfigRecord [TYPE Class] +# @defgroup Models Module group. # @BRIEF Stores persisted application configuration as a single authoritative record model. # @PRE SQLAlchemy declarative Base is initialized and table metadata registration is active. # @POST ORM table 'app_configurations' exposes id, payload, and updated_at fields with declared nullability/default semantics. @@ -29,6 +31,7 @@ class AppConfigRecord(Base): # #endregion AppConfigRecord # #region NotificationConfig [TYPE Class] +# @defgroup Models Module group. # @BRIEF Stores persisted provider-level notification configuration and encrypted credentials metadata. # @PRE SQLAlchemy declarative Base is initialized and uuid generation is available at instance creation time. # @POST ORM table 'notification_configs' exposes id, type, name, credentials, is_active, created_at, updated_at fields with declared constraints/defaults. diff --git a/backend/src/models/dashboard.py b/backend/src/models/dashboard.py index 37833a08..545482fe 100644 --- a/backend/src/models/dashboard.py +++ b/backend/src/models/dashboard.py @@ -1,4 +1,5 @@ # #region DashboardModels [C:3] [TYPE Module] [SEMANTICS pydantic, dashboard, model, schema, selection, dashboard-metadata] +# @defgroup Models Module group. # @BRIEF Defines data models for dashboard metadata and selection. # @LAYER Domain # @RELATION CALLED_BY -> [MigrationApi] diff --git a/backend/src/models/dataset_review.py b/backend/src/models/dataset_review.py index d10eefda..f56db056 100644 --- a/backend/src/models/dataset_review.py +++ b/backend/src/models/dataset_review.py @@ -1,4 +1,5 @@ # #region DatasetReviewModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, model, schema, facade] +# @defgroup Models Module group. # @BRIEF Thin facade re-exporting all dataset review domain models from the decomposed sub-package. # @LAYER Domain # @RELATION CALLS -> [DatasetReviewEnums] diff --git a/backend/src/models/dataset_review_pkg/__init__.py b/backend/src/models/dataset_review_pkg/__init__.py index c5269c4f..bcaec523 100644 --- a/backend/src/models/dataset_review_pkg/__init__.py +++ b/backend/src/models/dataset_review_pkg/__init__.py @@ -1,4 +1,5 @@ # #region DatasetReviewModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, model, schema, package] +# @defgroup Models Module group. # @BRIEF Re-export all dataset review domain models from decomposed sub-modules for backward-compatible imports. # @LAYER Domain diff --git a/backend/src/models/dataset_review_pkg/_clarification_models.py b/backend/src/models/dataset_review_pkg/_clarification_models.py index 2ef7ed76..fc303eee 100644 --- a/backend/src/models/dataset_review_pkg/_clarification_models.py +++ b/backend/src/models/dataset_review_pkg/_clarification_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewClarificationModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, clarification, model] +# @defgroup Models Module group. # @BRIEF Clarification session, question, option, and answer models for guided review flow. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -33,6 +34,7 @@ from src.models.mapping import Base # #region ClarificationSession [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One clarification session aggregate owning questions and tracking resolution progress. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class ClarificationSession(Base): @@ -57,6 +59,7 @@ class ClarificationSession(Base): # #region ClarificationQuestion [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One clarification question with priority ordering, options, and state machine. # @RELATION DEPENDS_ON -> [ClarificationSession] class ClarificationQuestion(Base): @@ -101,6 +104,7 @@ class ClarificationOption(Base): # #region ClarificationAnswer [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One persisted clarification answer with impact summary and feedback tracking. # @RELATION DEPENDS_ON -> [ClarificationQuestion] class ClarificationAnswer(Base): diff --git a/backend/src/models/dataset_review_pkg/_enums.py b/backend/src/models/dataset_review_pkg/_enums.py index dbe0dc60..85a413e1 100644 --- a/backend/src/models/dataset_review_pkg/_enums.py +++ b/backend/src/models/dataset_review_pkg/_enums.py @@ -1,4 +1,5 @@ # #region DatasetReviewEnums [C:2] [TYPE Module] [SEMANTICS enum, dataset, review, model, domain] +# @defgroup Models Module group. # @BRIEF All enumeration types for the dataset review domain, grouped for stable cross-module reuse. # @LAYER Domain # @INVARIANT Enum values are string-based for JSON serialization compatibility. diff --git a/backend/src/models/dataset_review_pkg/_execution_models.py b/backend/src/models/dataset_review_pkg/_execution_models.py index ea1465df..271bdf5d 100644 --- a/backend/src/models/dataset_review_pkg/_execution_models.py +++ b/backend/src/models/dataset_review_pkg/_execution_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewExecutionModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, execution, model] +# @defgroup Models Module group. # @BRIEF Compiled preview, run context, session event, and export artifact models for execution and audit. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -28,6 +29,7 @@ from src.models.mapping import Base # #region CompiledPreview [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One compiled SQL preview snapshot with fingerprint for staleness detection. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class CompiledPreview(Base): @@ -55,6 +57,7 @@ class CompiledPreview(Base): # #region DatasetRunContext [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Immutable launch audit snapshot capturing effective filters, template params, and approval state at launch time. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class DatasetRunContext(Base): @@ -84,6 +87,7 @@ class DatasetRunContext(Base): # #region SessionEvent [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One persisted audit event for dataset review session mutations. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class SessionEvent(Base): @@ -111,6 +115,7 @@ class SessionEvent(Base): # #region ExportArtifact [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One persisted export artifact reference for documentation and validation outputs. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class ExportArtifact(Base): diff --git a/backend/src/models/dataset_review_pkg/_filter_models.py b/backend/src/models/dataset_review_pkg/_filter_models.py index 9514fbce..f966ac28 100644 --- a/backend/src/models/dataset_review_pkg/_filter_models.py +++ b/backend/src/models/dataset_review_pkg/_filter_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewFilterModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, filter, template] +# @defgroup Models Module group. # @BRIEF Imported filter and template variable models for Superset context recovery and execution mapping. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -30,6 +31,7 @@ from src.models.mapping import Base # #region ImportedFilter [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Recovered Superset filter with confidence and recovery status tracking. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class ImportedFilter(Base): @@ -61,6 +63,7 @@ class ImportedFilter(Base): # #region TemplateVariable [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Discovered template variable from dataset SQL with mapping status tracking. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class TemplateVariable(Base): diff --git a/backend/src/models/dataset_review_pkg/_finding_models.py b/backend/src/models/dataset_review_pkg/_finding_models.py index 5095f139..aa955753 100644 --- a/backend/src/models/dataset_review_pkg/_finding_models.py +++ b/backend/src/models/dataset_review_pkg/_finding_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewFindingModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, finding, validation] +# @defgroup Models Module group. # @BRIEF Validation finding model for tracking blocking, warning, and informational issues during review. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -26,6 +27,7 @@ from src.models.mapping import Base # #region ValidationFinding [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Structured finding record for dataset review validation issues with resolution tracking. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class ValidationFinding(Base): diff --git a/backend/src/models/dataset_review_pkg/_mapping_models.py b/backend/src/models/dataset_review_pkg/_mapping_models.py index 20ee5cda..6a3f077d 100644 --- a/backend/src/models/dataset_review_pkg/_mapping_models.py +++ b/backend/src/models/dataset_review_pkg/_mapping_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewMappingModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, mapping, model] +# @defgroup Models Module group. # @BRIEF Execution mapping model linking imported filters to template variables with approval gates. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -28,6 +29,7 @@ from src.models.mapping import Base # #region ExecutionMapping [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One filter-to-variable mapping with approval gate, effective value, and transformation metadata. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @INVARIANT Explicit approval is required before launch when requires_explicit_approval is true. diff --git a/backend/src/models/dataset_review_pkg/_profile_models.py b/backend/src/models/dataset_review_pkg/_profile_models.py index 69295949..1357e0eb 100644 --- a/backend/src/models/dataset_review_pkg/_profile_models.py +++ b/backend/src/models/dataset_review_pkg/_profile_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewProfileModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, profile, summary] +# @defgroup Models Module group. # @BRIEF Dataset profile model capturing business summary, confidence, and completeness metadata. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -27,6 +28,7 @@ from src.models.mapping import Base # #region DatasetProfile [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One-to-one profile snapshot for a dataset review session, tracking business summary provenance and completeness. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class DatasetProfile(Base): diff --git a/backend/src/models/dataset_review_pkg/_semantic_models.py b/backend/src/models/dataset_review_pkg/_semantic_models.py index 8ee15780..0671b08d 100644 --- a/backend/src/models/dataset_review_pkg/_semantic_models.py +++ b/backend/src/models/dataset_review_pkg/_semantic_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewSemanticModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, semantic, enrichment] +# @defgroup Models Module group. # @BRIEF Semantic source, field entry, and candidate models for dictionary-driven semantic enrichment. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -38,6 +39,7 @@ from src.models.mapping import Base # #region SemanticSource [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Registered semantic enrichment source with trust level and application status. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class SemanticSource(Base): @@ -67,6 +69,7 @@ class SemanticSource(Base): # #region SemanticFieldEntry [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Per-field semantic metadata entry with provenance tracking, lock state, and candidate set. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [SemanticCandidate] @@ -109,6 +112,7 @@ class SemanticFieldEntry(Base): # #region SemanticCandidate [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF One proposed semantic value for a field entry, ranked by match type and confidence. # @RELATION DEPENDS_ON -> [SemanticFieldEntry] class SemanticCandidate(Base): diff --git a/backend/src/models/dataset_review_pkg/_session_models.py b/backend/src/models/dataset_review_pkg/_session_models.py index ce361eb3..a71da50e 100644 --- a/backend/src/models/dataset_review_pkg/_session_models.py +++ b/backend/src/models/dataset_review_pkg/_session_models.py @@ -1,4 +1,5 @@ # #region DatasetReviewSessionModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, dataset, review, session, model] +# @defgroup Models Module group. # @BRIEF Session aggregate root and collaborator models for dataset review orchestration. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums] @@ -33,6 +34,7 @@ from src.models.mapping import Base # #region SessionCollaborator [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF RBAC collaborator record linking a user to a dataset review session with a specific role. # @RELATION DEPENDS_ON -> [DatasetReviewSession] class SessionCollaborator(Base): @@ -54,6 +56,7 @@ class SessionCollaborator(Base): # #region DatasetReviewSession [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Aggregate root for the dataset review lifecycle, owning all child entities and driving readiness transitions. # @RELATION DEPENDS_ON -> [SessionCollaborator] # @RELATION DEPENDS_ON -> [DatasetProfile] diff --git a/backend/src/models/filter_state.py b/backend/src/models/filter_state.py index fdb394f3..21bd278b 100644 --- a/backend/src/models/filter_state.py +++ b/backend/src/models/filter_state.py @@ -1,4 +1,5 @@ # #region FilterStateModels [C:2] [TYPE Module] [SEMANTICS pydantic, filter, model, schema, superset, filter-state] +# @defgroup Models Module group. # # @BRIEF Pydantic models for Superset native filter state extraction and restoration. # @LAYER Domain @@ -10,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field # #region FilterState [C:2] [TYPE Model] +# @ingroup Models # @BRIEF Represents the state of a single native filter. # @DATA_CONTRACT Input[extraFormData: Dict, filterState: Dict, ownState: Optional[Dict]] -> Model[FilterState] class FilterState(BaseModel): @@ -24,6 +26,7 @@ class FilterState(BaseModel): # #region NativeFilterDataMask [C:2] [TYPE Model] +# @ingroup Models # @BRIEF Represents the dataMask containing all native filter states. # @DATA_CONTRACT Input[Dict[filter_id, FilterState]] -> Model[NativeFilterDataMask] class NativeFilterDataMask(BaseModel): @@ -47,6 +50,7 @@ class NativeFilterDataMask(BaseModel): # #region ParsedNativeFilters [C:2] [TYPE Model] +# @ingroup Models # @BRIEF Result of parsing native filters from permalink or native_filters_key. # @DATA_CONTRACT Input[dataMask: Dict, metadata: Dict] -> Model[ParsedNativeFilters] class ParsedNativeFilters(BaseModel): @@ -74,6 +78,7 @@ class ParsedNativeFilters(BaseModel): # #region DashboardURLFilterExtraction [C:2] [TYPE Model] +# @ingroup Models # @BRIEF Result of parsing a complete dashboard URL for filter information. # @DATA_CONTRACT Input[url: str, dashboard_id: Optional, filter_type: Optional, filters: Dict] -> Model[DashboardURLFilterExtraction] class DashboardURLFilterExtraction(BaseModel): @@ -91,6 +96,7 @@ class DashboardURLFilterExtraction(BaseModel): # #region ExtraFormDataMerge [C:2] [TYPE Model] +# @ingroup Models # @BRIEF Configuration for merging extraFormData from different sources. # @DATA_CONTRACT Input[append_keys: List[str], override_keys: List[str]] -> Model[ExtraFormDataMerge] class ExtraFormDataMerge(BaseModel): diff --git a/backend/src/models/git.py b/backend/src/models/git.py index 62a22192..165b2565 100644 --- a/backend/src/models/git.py +++ b/backend/src/models/git.py @@ -1,4 +1,5 @@ # #region GitModels [TYPE Module] [SEMANTICS sqlalchemy, git, model, config, sync] +# @defgroup Models Module group. from datetime import UTC, datetime import enum diff --git a/backend/src/models/llm.py b/backend/src/models/llm.py index 0a0d5eae..19722955 100644 --- a/backend/src/models/llm.py +++ b/backend/src/models/llm.py @@ -1,4 +1,5 @@ # #region LlmModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, llm, model, schema, validate, provider] +# @defgroup Models Module group. # @BRIEF SQLAlchemy models for LLM provider configuration and validation results. # @LAYER Domain # @RELATION INHERITS -> [MappingModels] @@ -17,6 +18,7 @@ def generate_uuid(): # #region ValidationPolicy [TYPE Class] +# @defgroup Models Module group. # @BRIEF Defines a scheduled rule for validating a group of dashboards within an execution window. # @RELATION DEPENDS_ON -> [LLMProvider] class ValidationPolicy(Base): @@ -53,6 +55,7 @@ class ValidationPolicy(Base): # #region LLMProvider [TYPE Class] +# @defgroup Models Module group. # @BRIEF SQLAlchemy model for LLM provider configuration. class LLMProvider(Base): __tablename__ = "llm_providers" @@ -79,6 +82,7 @@ class LLMProvider(Base): # #region ValidationSource [TYPE Class] +# @defgroup Models Module group. # @BRIEF Relational source mapping for validation policies (v2). class ValidationSource(Base): __tablename__ = "validation_sources" @@ -100,6 +104,7 @@ class ValidationSource(Base): # #region ValidationRecord [TYPE Class] +# @defgroup Models Module group. # @BRIEF SQLAlchemy model for dashboard validation history. class ValidationRecord(Base): __tablename__ = "llm_validation_results" @@ -131,6 +136,7 @@ class ValidationRecord(Base): # #region ValidationRun [TYPE Class] +# @defgroup Models Module group. # @BRIEF Tracks a single validation execution run (v2). class ValidationRun(Base): __tablename__ = "validation_runs" diff --git a/backend/src/models/maintenance.py b/backend/src/models/maintenance.py index a4b9b5c7..814b5ff3 100644 --- a/backend/src/models/maintenance.py +++ b/backend/src/models/maintenance.py @@ -1,4 +1,5 @@ # #region MaintenanceModels [C:2] [TYPE Module] [SEMANTICS sqlalchemy, maintenance, banner, event, settings, model] +# @defgroup Models Module group. # @BRIEF SQLAlchemy models for the Maintenance Banner feature: event, banner, dashboard state, and settings. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/models/mapping.py b/backend/src/models/mapping.py index 0c9961a1..7e16e1ec 100644 --- a/backend/src/models/mapping.py +++ b/backend/src/models/mapping.py @@ -1,4 +1,5 @@ # #region MappingModels [C:5] [TYPE Module] [SEMANTICS sqlalchemy, mapping, model, schema, resource-type] +# @defgroup Models Module group. # # @BRIEF Defines the database schema for environment metadata and database mappings using SQLAlchemy. # @LAYER Domain @@ -44,6 +45,7 @@ class MigrationStatus(enum.Enum): # #endregion MigrationStatus # #region Environment [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a Superset instance environment. # @RELATION DEPENDS_ON -> MappingModels class Environment(Base): @@ -56,6 +58,7 @@ class Environment(Base): # #endregion Environment # #region DatabaseMapping [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a mapping between source and target databases. class DatabaseMapping(Base): __tablename__ = "database_mappings" @@ -71,6 +74,7 @@ class DatabaseMapping(Base): # #endregion DatabaseMapping # #region MigrationJob [C:2] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a single migration execution job. class MigrationJob(Base): __tablename__ = "migration_jobs" @@ -84,6 +88,7 @@ class MigrationJob(Base): # #endregion MigrationJob # #region ResourceMapping [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Maps a universal UUID for a resource to its actual ID on a specific environment. # @TEST_DATA: resource_mapping_record -> {'environment_id': 'prod-env-1', 'resource_type': 'chart', 'uuid': '123e4567-e89b-12d3-a456-426614174000', 'remote_integer_id': '42'} # @RELATION DEPENDS_ON -> MappingModels diff --git a/backend/src/models/profile.py b/backend/src/models/profile.py index 9b85ef75..62808663 100644 --- a/backend/src/models/profile.py +++ b/backend/src/models/profile.py @@ -1,4 +1,5 @@ # #region ProfileModels [C:5] [TYPE Module] [SEMANTICS sqlalchemy, profile, model, schema, git, dashboard] +# @defgroup Models Module group. # # @BRIEF Defines persistent per-user profile settings for dashboard filter, Git identity/token, and UX preferences. # @LAYER Domain @@ -22,6 +23,7 @@ from .mapping import Base # #region UserDashboardPreference [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Stores Superset username binding and default "my dashboards" toggle for one authenticated user. # @RELATION INHERITS -> [MappingModels] class UserDashboardPreference(Base): diff --git a/backend/src/models/report.py b/backend/src/models/report.py index 4840907e..2555d0e0 100644 --- a/backend/src/models/report.py +++ b/backend/src/models/report.py @@ -1,4 +1,5 @@ # #region ReportModels [C:3] [TYPE Module] [SEMANTICS pydantic, report, model, schema, task, task-type] +# @defgroup Models Module group. # @BRIEF Canonical report schemas for unified task reporting across heterogeneous task types. # @LAYER Domain # @PRE Pydantic library and task manager models are available. @@ -16,6 +17,7 @@ from pydantic import BaseModel, Field, field_validator, model_validator # #region TaskType [C:3] [TYPE Class] [SEMANTICS enum, type, task] +# @defgroup Models Module group. # @INVARIANT Must contain valid generic task type mappings. # @RELATION DEPENDS_ON -> ReportModels # @BRIEF Supported normalized task report types. @@ -32,6 +34,7 @@ class TaskType(str, Enum): # #region ReportStatus [C:3] [TYPE Class] [SEMANTICS enum, status, task] +# @defgroup Models Module group. # @INVARIANT TaskStatus enum mapping logic holds. # @BRIEF Supported normalized report status values. # @RELATION DEPENDS_ON -> ReportModels @@ -46,6 +49,7 @@ class ReportStatus(str, Enum): # #region ErrorContext [C:3] [TYPE Class] [SEMANTICS error, context, payload] +# @defgroup Models Module group. # @INVARIANT The properties accurately describe error state. # @BRIEF Error and recovery context for failed/partial reports. # @@ -72,6 +76,7 @@ class ErrorContext(BaseModel): # #region TaskReport [C:3] [TYPE Class] [SEMANTICS report, model, summary] +# @defgroup Models Module group. # @INVARIANT Must represent canonical task record attributes. # @BRIEF Canonical normalized report envelope for one task execution. # @@ -130,6 +135,7 @@ class TaskReport(BaseModel): # #region ReportQuery [C:3] [TYPE Class] [SEMANTICS query, filter, search] +# @defgroup Models Module group. # @INVARIANT Time and pagination queries are mutually consistent. # @BRIEF Query object for server-side report filtering, sorting, and pagination. # @@ -189,6 +195,7 @@ class ReportQuery(BaseModel): # #region ReportCollection [C:3] [TYPE Class] [SEMANTICS collection, pagination] +# @defgroup Models Module group. # @INVARIANT Represents paginated data correctly. # @BRIEF Paginated collection of normalized task reports. # @@ -215,6 +222,7 @@ class ReportCollection(BaseModel): # #region ReportDetailView [C:3] [TYPE Class] [SEMANTICS view, detail, logs] +# @defgroup Models Module group. # @INVARIANT Incorporates a report and logs correctly. # @BRIEF Detailed report representation including diagnostics and recovery actions. # diff --git a/backend/src/models/storage.py b/backend/src/models/storage.py index 6af947f8..522cedb9 100644 --- a/backend/src/models/storage.py +++ b/backend/src/models/storage.py @@ -1,4 +1,5 @@ # #region StorageModels [TYPE Module] [SEMANTICS pydantic, storage, model, backup, config] +# @defgroup Models Module group. # @RATIONALE Fixed typo 'repositorys' → 'repositories' in FileCategory.REPOSITORY and StorageConfig.repo_path default. Breaking change documented for existing installations. from datetime import datetime from enum import Enum diff --git a/backend/src/models/task.py b/backend/src/models/task.py index 62d41738..3b667ef4 100644 --- a/backend/src/models/task.py +++ b/backend/src/models/task.py @@ -33,6 +33,7 @@ class TaskRecord(Base): # #endregion TaskRecord # #region TaskLogRecord [C:3] [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a single persistent log entry for a task. # @RELATION DEPENDS_ON -> TaskRecord # @INVARIANT Each log entry belongs to exactly one task. diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index b5372aad..963aa9ba 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -1,4 +1,5 @@ # #region TranslateModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, model, schema, dashboard, llm] +# @defgroup Models Module group. # @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects. # @LAYER Domain # @RELATION INHERITS -> [MappingModels] @@ -17,6 +18,7 @@ def generate_uuid(): # #region TranslationJob [TYPE Class] +# @defgroup Models Module group. # @BRIEF A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments. # @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. # @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity. @@ -66,6 +68,7 @@ class TranslationJob(Base): # #region TranslationRun [TYPE Class] +# @defgroup Models Module group. # @BRIEF Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata. class TranslationRun(Base): __tablename__ = "translation_runs" @@ -97,6 +100,7 @@ class TranslationRun(Base): # #region TranslationBatch [TYPE Class] +# @defgroup Models Module group. # @BRIEF Groups translation records within a run into manageable batches with timing and record counts. class TranslationBatch(Base): __tablename__ = "translation_batches" @@ -115,6 +119,7 @@ class TranslationBatch(Base): # #region TranslationRecord [TYPE Class] +# @defgroup Models Module group. # @BRIEF Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state. class TranslationRecord(Base): __tablename__ = "translation_records" @@ -148,6 +153,7 @@ class TranslationRecord(Base): # #region TranslationEvent [TYPE Class] +# @defgroup Models Module group. # @BRIEF Audit/event log for translation operations, with optional run_id for context. class TranslationEvent(Base): __tablename__ = "translation_events" @@ -163,6 +169,7 @@ class TranslationEvent(Base): # #region TranslationPreviewSession [TYPE Class] +# @defgroup Models Module group. # @BRIEF A preview session allowing users to review proposed translations before applying them. class TranslationPreviewSession(Base): __tablename__ = "translation_preview_sessions" @@ -178,6 +185,7 @@ class TranslationPreviewSession(Base): # #region TranslationPreviewRecord [TYPE Class] +# @defgroup Models Module group. # @BRIEF Individual preview entry within a preview session, showing original and translated content side by side. class TranslationPreviewRecord(Base): __tablename__ = "translation_preview_records" @@ -199,6 +207,7 @@ class TranslationPreviewRecord(Base): # #region TerminologyDictionary [TYPE Class] +# @defgroup Models Module group. # @BRIEF A named collection of terminology mappings used during translation to ensure consistent term translation. class TerminologyDictionary(Base): __tablename__ = "terminology_dictionaries" @@ -214,6 +223,7 @@ class TerminologyDictionary(Base): # #region DictionaryEntry [TYPE Class] +# @defgroup Models Module group. # @BRIEF A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking. class DictionaryEntry(Base): __tablename__ = "dictionary_entries" @@ -250,6 +260,7 @@ class DictionaryEntry(Base): # #region TranslationSchedule [TYPE Class] +# @defgroup Models Module group. # @BRIEF Defines a cron-based schedule for recurring translation jobs. class TranslationSchedule(Base): __tablename__ = "translation_schedules" @@ -269,6 +280,7 @@ class TranslationSchedule(Base): # #region TranslationJobDictionary [TYPE Class] +# @defgroup Models Module group. # @BRIEF Many-to-many association between translation jobs and terminology dictionaries. class TranslationJobDictionary(Base): __tablename__ = "translation_job_dictionaries" @@ -288,6 +300,7 @@ class TranslationJobDictionary(Base): # #region MetricSnapshot [TYPE Class] +# @defgroup Models Module group. # @BRIEF Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation. # @RATIONALE job_id is nullable with SET NULL because prune_expired creates aggregate snapshots # (job_id=None) that span all jobs — enforced FK in PostgreSQL rejects virtual IDs like diff --git a/backend/src/plugins/__init__.py b/backend/src/plugins/__init__.py index 828de331..64f94237 100644 --- a/backend/src/plugins/__init__.py +++ b/backend/src/plugins/__init__.py @@ -1,3 +1,4 @@ # #region src.plugins [TYPE Package] [SEMANTICS plugin, package, discovery, runtime] +# @ingroup Plugin # @BRIEF Plugin package root for dynamic discovery and runtime imports. # #endregion src.plugins diff --git a/backend/src/plugins/backup.py b/backend/src/plugins/backup.py index 2e218c0b..056f2cf5 100755 --- a/backend/src/plugins/backup.py +++ b/backend/src/plugins/backup.py @@ -1,4 +1,5 @@ # #region BackupPlugin [TYPE Module] [SEMANTICS backup, export, archive, superset, dashboard] +# @defgroup Plugin Module group. # @BRIEF A plugin that provides functionality to back up Superset dashboards. # @LAYER App # @RELATION IMPLEMENTS -> PluginBase @@ -22,6 +23,7 @@ from ..dependencies import get_config_manager # #region BackupPlugin [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Implementation of the backup plugin logic. class BackupPlugin(PluginBase): """ diff --git a/backend/src/plugins/debug.py b/backend/src/plugins/debug.py index a739c979..a2438789 100644 --- a/backend/src/plugins/debug.py +++ b/backend/src/plugins/debug.py @@ -1,4 +1,5 @@ # #region DebugPluginModule [TYPE Module] [SEMANTICS debug, diagnostic, superset, api, system] +# @defgroup Plugin Module group. # @BRIEF Implements a plugin for system diagnostics and debugging Superset API responses. # @LAYER Plugin # @BRIEF Plugin for diagnostics. Inherits PluginBase. @@ -13,6 +14,7 @@ from ..core.utils.client_registry import get_superset_client # #region DebugPlugin [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Plugin for system diagnostics and debugging. class DebugPlugin(PluginBase): """ diff --git a/backend/src/plugins/git/__init__.py b/backend/src/plugins/git/__init__.py index 61671d14..fd70741d 100644 --- a/backend/src/plugins/git/__init__.py +++ b/backend/src/plugins/git/__init__.py @@ -1,3 +1,4 @@ # #region GitPluginExt [TYPE Package] [SEMANTICS git, plugin, extension, package] +# @ingroup Git # @BRIEF Git plugin extension package root. # #endregion GitPluginExt diff --git a/backend/src/plugins/git/llm_extension.py b/backend/src/plugins/git/llm_extension.py index 278cce94..27d06839 100644 --- a/backend/src/plugins/git/llm_extension.py +++ b/backend/src/plugins/git/llm_extension.py @@ -1,4 +1,5 @@ # #region GitLLMExtensionModule [C:3] [TYPE Module] [SEMANTICS git, llm, commit, message, generation] +# @defgroup Git Module group. # @BRIEF LLM-based extensions for the Git plugin, specifically for commit message generation. # @LAYER Domain # @RELATION DEPENDS_ON -> [LLMClient] @@ -12,6 +13,7 @@ from ..llm_analysis.service import LLMClient # #region GitLLMExtension [TYPE Class] +# @defgroup Git Module group. # @BRIEF Provides LLM capabilities to the Git plugin. class GitLLMExtension: def __init__(self, client: LLMClient): diff --git a/backend/src/plugins/git_plugin.py b/backend/src/plugins/git_plugin.py index b5620c55..90205f00 100644 --- a/backend/src/plugins/git_plugin.py +++ b/backend/src/plugins/git_plugin.py @@ -1,4 +1,5 @@ # #region GitPluginModule [C:4] [TYPE Module] [SEMANTICS git, versioning, deploy, sync, superset, backup, transactional] +# @defgroup Plugin Module group. # # @BRIEF Предоставляет плагин для версионирования и развертывания дашбордов Superset. # @LAYER Plugin @@ -123,6 +124,7 @@ def _pack_deploy_zip(repo_path: Path, root_dir_name: str, zip_buffer: io.BytesIO # #region GitPlugin [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Реализация плагина Git Integration для управления версиями дашбордов. class GitPlugin(PluginBase): diff --git a/backend/src/plugins/llm_analysis/__init__.py b/backend/src/plugins/llm_analysis/__init__.py index b286d434..446f5d28 100644 --- a/backend/src/plugins/llm_analysis/__init__.py +++ b/backend/src/plugins/llm_analysis/__init__.py @@ -1,4 +1,5 @@ # #region LLMAnalysisPackage [TYPE Module] [SEMANTICS llm, analysis, plugin, package, export] +# @defgroup LLMAnalysis Module group. """ LLM Analysis Plugin for automated dashboard validation and dataset documentation. diff --git a/backend/src/plugins/llm_analysis/_topology.py b/backend/src/plugins/llm_analysis/_topology.py index 4400d792..0a6f7b38 100644 --- a/backend/src/plugins/llm_analysis/_topology.py +++ b/backend/src/plugins/llm_analysis/_topology.py @@ -1,4 +1,5 @@ # #region DashboardTopology [C:3] [TYPE Module] [SEMANTICS superset, topology, dashboard, layout, parsing] +# @defgroup LLMAnalysis Module group. # @BRIEF Dashboard topology extraction — parses Superset position_json into human-readable # text descriptions of Tab → Row → Chart hierarchy for LLM prompt context. # @LAYER Plugin @@ -227,6 +228,7 @@ def _extract_topology_children( # #region build_dashboard_topology [C:2] [TYPE Function] +# @ingroup LLMAnalysis # @BRIEF Build a structured text description of dashboard layout from position_json. # @PARAM dashboard (dict) - Dashboard metadata dict with position_json, dashboard_title. # @PARAM charts (list[dict]) - List of fetched chart detail dicts for name/viz lookup. diff --git a/backend/src/plugins/llm_analysis/migrations/v1_to_v2.py b/backend/src/plugins/llm_analysis/migrations/v1_to_v2.py index b008cc5e..3796297f 100644 --- a/backend/src/plugins/llm_analysis/migrations/v1_to_v2.py +++ b/backend/src/plugins/llm_analysis/migrations/v1_to_v2.py @@ -1,4 +1,5 @@ # #region V1ToV2Migration [C:3] [TYPE Module] [SEMANTICS migration,validation,data,idempotent] +# @defgroup LLMAnalysis Module group. # @BRIEF Migration script: v1 → v2 ValidationPolicy data. Converts existing dashboard_ids # JSON arrays to ValidationSource relational rows. # @PRE Database has both validation_policies and validation_sources tables. diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index 6e289018..a02089dc 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -1,4 +1,5 @@ # #region LLMAnalysisModels [C:3] [TYPE Module] [SEMANTICS pydantic, llm, plugin, model, provider-type] +# @defgroup LLMAnalysis Module group. # @BRIEF Define Pydantic models for LLM Analysis plugin. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] @@ -10,6 +11,7 @@ from pydantic import BaseModel, Field # #region LLMProviderType [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Enum for supported LLM providers. class LLMProviderType(str, Enum): OPENAI = "openai" @@ -19,6 +21,7 @@ class LLMProviderType(str, Enum): # #endregion LLMProviderType # #region LLMProviderConfig [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Configuration for an LLM provider. class LLMProviderConfig(BaseModel): id: str | None = None @@ -41,6 +44,7 @@ class LLMProviderConfig(BaseModel): # #endregion LLMProviderConfig # #region ValidationStatus [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Enum for dashboard validation status. class ValidationStatus(str, Enum): PASS = "PASS" @@ -50,6 +54,7 @@ class ValidationStatus(str, Enum): # #endregion ValidationStatus # #region DetectedIssue [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Model for a single issue detected during validation. class DetectedIssue(BaseModel): severity: ValidationStatus @@ -58,6 +63,7 @@ class DetectedIssue(BaseModel): # #endregion DetectedIssue # #region ValidationResult [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Model for dashboard validation result. class ValidationResult(BaseModel): id: str | None = None diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index c694da83..b20daa5b 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -1,4 +1,5 @@ # #region LLMAnalysisPlugin [C:5] [TYPE Module] [SEMANTICS llm, analysis, dashboard, validation, documentation] +# @defgroup LLMAnalysis Module group. # @BRIEF Implements DashboardValidationPlugin and DocumentationPlugin. # @LAYER Plugin # @RELATION INHERITS -> [PluginBase] @@ -159,6 +160,7 @@ def _update_run_status(db, run_id: str | None) -> None: # #region DashboardValidationPlugin [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Plugin for automated dashboard health analysis using LLMs. # @RELATION IMPLEMENTS -> [PluginBase] class DashboardValidationPlugin(PluginBase): @@ -846,6 +848,7 @@ class DashboardValidationPlugin(PluginBase): # #region DocumentationPlugin [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Plugin for automated dataset documentation using LLMs. # @RELATION IMPLEMENTS -> [PluginBase] class DocumentationPlugin(PluginBase): diff --git a/backend/src/plugins/llm_analysis/scheduler.py b/backend/src/plugins/llm_analysis/scheduler.py index 5af038b9..115257b7 100644 --- a/backend/src/plugins/llm_analysis/scheduler.py +++ b/backend/src/plugins/llm_analysis/scheduler.py @@ -1,4 +1,5 @@ # #region LLMAnalysisScheduler [C:3] [TYPE Module] [SEMANTICS scheduler, llm, validation, task, cron] +# @defgroup LLMAnalysis Module group. # @BRIEF Provides helper functions to schedule LLM-based validation tasks. # @LAYER Domain # @RELATION DEPENDS_ON -> [SchedulerService] @@ -10,6 +11,7 @@ from ...dependencies import get_scheduler_service, get_task_manager # #region schedule_dashboard_validation [TYPE Function] +# @ingroup LLMAnalysis # @BRIEF Schedules a recurring dashboard validation task. # @SIDE_EFFECT Adds a job to the scheduler service. def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: dict[str, Any]): diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index d54feefa..58e807ba 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -1,4 +1,5 @@ # #region LLMAnalysisService [C:5] [TYPE Module] [SEMANTICS llm, screenshot, playwright, openai, tenacity] +# @defgroup LLMAnalysis Module group. # @BRIEF Services for LLM interaction and dashboard screenshots. # @LAYER Plugin # @RELATION DEPENDS_ON -> [EXT:Library:tenacity] @@ -37,6 +38,7 @@ LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout) DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" # #region ScreenshotService [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Handles capturing screenshots of Superset dashboards. class ScreenshotService: # region ScreenshotService.__init__ [TYPE Function] @@ -837,6 +839,7 @@ class ScreenshotService: # #endregion ScreenshotService # #region LLMClient [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Wrapper for LLM provider APIs. class LLMClient: # region LLMClient.__init__ [TYPE Function] @@ -1441,6 +1444,7 @@ class LLMClient: # #endregion LLMClient # #region DatasetHealthChecker [C:3] [TYPE Class] +# @defgroup LLMAnalysis Module group. # @BRIEF Checks dataset accessibility and KXD connectivity via Superset API. # @LAYER Service # @RELATION CALLS -> [SupersetClient] @@ -1641,6 +1645,7 @@ class DatasetHealthChecker: # #endregion DatasetHealthChecker # #region RedactionService [C:2] [TYPE Module] +# @defgroup LLMAnalysis Module group. # @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses. # @LAYER Service # @RATIONALE FR-029: sensitive data must be filtered BEFORE external LLM send and BEFORE persistence. diff --git a/backend/src/plugins/maintenance_banner.py b/backend/src/plugins/maintenance_banner.py index 774dea5e..9552eee7 100644 --- a/backend/src/plugins/maintenance_banner.py +++ b/backend/src/plugins/maintenance_banner.py @@ -1,4 +1,5 @@ # #region MaintenanceBannerPlugin [C:4] [TYPE Module] [SEMANTICS plugin, maintenance, banner, execution, task] +# @defgroup Plugin Module group. # @BRIEF TaskManager plugin for executing maintenance banner operations (start, end, end-all). # Dispatches to MaintenanceService based on operation type in params. # @LAYER Plugin @@ -21,6 +22,7 @@ from ..services.maintenance import end_all_maintenance, end_maintenance, start_m # #region MaintenanceBannerPlugin [C:4] [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Plugin for executing maintenance banner operations asynchronously via TaskManager. # @RELATION CALLS -> [start_maintenance] # @RELATION CALLS -> [end_maintenance] @@ -29,36 +31,42 @@ class MaintenanceBannerPlugin(PluginBase): """TaskManager plugin for maintenance banner operations.""" # #region id [TYPE Property] +# @ingroup Plugin @property def id(self) -> str: return "maintenance_banner_apply" # #endregion id # #region name [TYPE Property] +# @ingroup Plugin @property def name(self) -> str: return "Maintenance Banner" # #endregion name # #region description [TYPE Property] +# @ingroup Plugin @property def description(self) -> str: return "Applies or removes maintenance banners on Superset dashboards." # #endregion description # #region version [TYPE Property] +# @ingroup Plugin @property def version(self) -> str: return "1.0.0" # #endregion version # #region ui_route [TYPE Property] +# @ingroup Plugin @property def ui_route(self) -> str: return "" # No UI route — async-only # #endregion ui_route # #region get_schema [TYPE Function] +# @ingroup Plugin def get_schema(self) -> dict[str, Any]: return { "type": "object", @@ -82,6 +90,7 @@ class MaintenanceBannerPlugin(PluginBase): # #endregion get_schema # #region execute [TYPE Function] [C:4] +# @ingroup Plugin # @BRIEF Execute maintenance operation based on params.operation. # @PARAM params contains: operation (start|end|end_all), event_id, environment_id. # @PARAM context TaskContext for logging. diff --git a/backend/src/plugins/mapper.py b/backend/src/plugins/mapper.py index 3adb4450..38a8c802 100644 --- a/backend/src/plugins/mapper.py +++ b/backend/src/plugins/mapper.py @@ -1,4 +1,5 @@ # #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, sqllab, excel, mapping] +# @defgroup Plugin Module group. # @BRIEF Implements a plugin for mapping dataset columns using Superset SQL Lab or Excel files. # @LAYER Plugin # @BRIEF Plugin for dataset column mapping. Inherits PluginBase. @@ -14,6 +15,7 @@ from ..core.utils.dataset_mapper import DatasetMapper # #region MapperPlugin [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Plugin for mapping dataset columns verbose names. class MapperPlugin(PluginBase): """ diff --git a/backend/src/plugins/migration.py b/backend/src/plugins/migration.py index 6a969487..b6e2a773 100755 --- a/backend/src/plugins/migration.py +++ b/backend/src/plugins/migration.py @@ -1,4 +1,5 @@ # #region MigrationPlugin [C:5] [TYPE Module] [SEMANTICS migration, export, import, mapping, superset] +# @defgroup Plugin Module group. # @BRIEF Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments. # @LAYER App # @RELATION IMPLEMENTS -> PluginBase @@ -28,6 +29,7 @@ from ..models.mapping import DatabaseMapping, Environment # #region MigrationPlugin [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Implementation of the migration plugin workflow and transformation orchestration. # @PRE SupersetClient authenticated, database session active # @POST Returns MigrationResult with success/failure status and artifact list diff --git a/backend/src/plugins/search.py b/backend/src/plugins/search.py index bddc828e..8bec5674 100644 --- a/backend/src/plugins/search.py +++ b/backend/src/plugins/search.py @@ -1,4 +1,5 @@ # #region SearchPluginModule [TYPE Module] [SEMANTICS search, dataset, text, pattern, superset] +# @defgroup Plugin Module group. # @BRIEF Implements a plugin for searching text patterns across all datasets in a specific Superset environment. # @LAYER Plugin # @BRIEF Plugin for text search across datasets. Inherits PluginBase. @@ -14,6 +15,7 @@ from ..core.task_manager.context import TaskContext # #region SearchPlugin [TYPE Class] +# @defgroup Plugin Module group. # @BRIEF Plugin for searching text patterns in Superset datasets. class SearchPlugin(PluginBase): """ diff --git a/backend/src/plugins/storage/__init__.py b/backend/src/plugins/storage/__init__.py index d8372dca..6daee536 100644 --- a/backend/src/plugins/storage/__init__.py +++ b/backend/src/plugins/storage/__init__.py @@ -1,4 +1,5 @@ # #region StoragePluginPackage [TYPE Package] [SEMANTICS storage, plugin, package, export] +# @ingroup Storage from .plugin import StoragePlugin __all__ = ["StoragePlugin"] diff --git a/backend/src/plugins/storage/plugin.py b/backend/src/plugins/storage/plugin.py index be559356..d47d712a 100644 --- a/backend/src/plugins/storage/plugin.py +++ b/backend/src/plugins/storage/plugin.py @@ -1,4 +1,5 @@ # #region StoragePlugin [TYPE Module] [SEMANTICS fastapi, storage, filesystem, backup, archive] +# @defgroup Storage Module group. # @BRIEF Provides core filesystem operations for managing backups and repositories. # @LAYER App # @RELATION DEPENDS_ON -> [TaskContext] @@ -34,6 +35,7 @@ def _copy_upload_to_disk(dest_path: Path, file_obj) -> None: # #region StoragePlugin [TYPE Class] +# @defgroup Storage Module group. # @BRIEF Implementation of the storage management plugin. class StoragePlugin(PluginBase): """ diff --git a/backend/src/plugins/translate/__init__.py b/backend/src/plugins/translate/__init__.py index fd8f88d4..6a012609 100644 --- a/backend/src/plugins/translate/__init__.py +++ b/backend/src/plugins/translate/__init__.py @@ -1,2 +1,3 @@ # #region TranslatePluginPackage [TYPE Package] [SEMANTICS translate, plugin, package, sql, dashboard] +# @ingroup Translate # #endregion TranslatePluginPackage diff --git a/backend/src/plugins/translate/__tests__/test_preview.py b/backend/src/plugins/translate/__tests__/test_preview.py index 6c9af843..9d75ce8d 100644 --- a/backend/src/plugins/translate/__tests__/test_preview.py +++ b/backend/src/plugins/translate/__tests__/test_preview.py @@ -7,14 +7,15 @@ from datetime import UTC, datetime, timedelta import json import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from src.models.translate import ( TranslationJob, TranslationPreviewRecord, TranslationPreviewSession, ) -from src.plugins.translate.preview import TokenEstimator, TranslationPreview +from src.plugins.translate.preview import TranslationPreview +from src.plugins.translate.preview_token_estimator import TokenEstimator from src.plugins.translate.preview_executor import PreviewExecutor @@ -59,7 +60,8 @@ class TestTranslationPreview: # region test_preview_valid_job [TYPE Function] # @PURPOSE: Verify preview creates session and records successfully. - def test_preview_valid_job(self): + @pytest.mark.asyncio + async def test_preview_valid_job(self): """Preview with valid job should create session and return records.""" db = MagicMock() config_manager = MagicMock() @@ -79,21 +81,21 @@ class TestTranslationPreview: config_manager.get_environments.return_value = [env] # Mock SupersetClient - with patch("src.plugins.translate.preview_executor.SupersetClient") as MockClient: + with patch("src.plugins.translate.preview_executor.get_superset_client") as MockClient: mock_client = MagicMock() MockClient.return_value = mock_client - # Mock dataset_detail - mock_client.get_dataset_detail.return_value = { + # Mock dataset_detail (async method) + mock_client.get_dataset_detail = AsyncMock(return_value={ "table_name": "test_table", "columns": [ {"column_name": "title", "type": "VARCHAR"}, {"column_name": "category", "type": "VARCHAR"}, {"column_name": "description", "type": "TEXT"}, ], - } + }) - # Mock build_dataset_preview_query_context + # Mock build_dataset_preview_query_context (NOT async) mock_client.build_dataset_preview_query_context.return_value = { "datasource": {"id": 42, "type": "table"}, "queries": [{ @@ -107,8 +109,8 @@ class TestTranslationPreview: "force": True, } - # Mock network request to Superset - mock_client.network.request.return_value = { + # Mock network request to Superset (async) + mock_client.network.request = AsyncMock(return_value={ "result": [{ "status": "success", "data": [ @@ -116,7 +118,7 @@ class TestTranslationPreview: {"title": "Goodbye", "category": "farewell", "description": "A parting message"}, ], }] - } + }) # Mock LLM provider with patch("src.services.llm_provider.LLMProviderService") as MockProviderService: @@ -140,7 +142,7 @@ class TestTranslationPreview: mock_filter.return_value = [] preview_service = TranslationPreview(db, config_manager, "test-user") - result = preview_service.preview_rows(job_id="job-123", sample_size=10) + result = await preview_service.preview_rows(job_id="job-123", sample_size=10) # Verify result structure assert result["job_id"] == "job-123" @@ -165,7 +167,8 @@ class TestTranslationPreview: # region test_preview_with_dictionary [TYPE Function] # @PURPOSE: Verify glossary entries from dictionary are included in LLM prompt. - def test_preview_with_dictionary(self): + @pytest.mark.asyncio + async def test_preview_with_dictionary(self): """Preview should include dictionary glossary in the prompt sent to LLM.""" db = MagicMock() config_manager = MagicMock() @@ -177,10 +180,10 @@ class TestTranslationPreview: env.id = "postgresql" config_manager.get_environments.return_value = [env] - with patch("src.plugins.translate.preview_executor.SupersetClient") as MockClient: + with patch("src.plugins.translate.preview_executor.get_superset_client") as MockClient: mock_client = MagicMock() MockClient.return_value = mock_client - mock_client.get_dataset_detail.return_value = {"table_name": "test", "columns": []} + mock_client.get_dataset_detail = AsyncMock(return_value={"table_name": "test", "columns": []}) mock_client.build_dataset_preview_query_context.return_value = { "datasource": {"id": 42, "type": "table"}, "queries": [{"columns": [], "metrics": ["count"], "row_limit": 10}], @@ -189,9 +192,9 @@ class TestTranslationPreview: "result_type": "query", "force": True, } - mock_client.network.request.return_value = { + mock_client.network.request = AsyncMock(return_value={ "result": [{"status": "success", "data": [{"title": "Hello World", "category": "greeting"}]}] - } + }) with patch("src.services.llm_provider.LLMProviderService") as MockProviderService: mock_provider_svc = MagicMock() @@ -215,7 +218,7 @@ class TestTranslationPreview: ] preview_service = TranslationPreview(db, config_manager, "test-user") - result = preview_service.preview_rows(job_id="job-123", sample_size=5) + result = await preview_service.preview_rows(job_id="job-123", sample_size=5) # Verify LLM was called mock_llm.assert_called_once() @@ -508,7 +511,8 @@ class TestTranslationPreview: # region test_preview_missing_datasource [TYPE Function] # @PURPOSE: Verify error when job has no datasource configured. - def test_preview_missing_datasource(self): + @pytest.mark.asyncio + async def test_preview_missing_datasource(self): """Preview should fail if job has no datasource.""" db = MagicMock() config_manager = MagicMock() @@ -518,12 +522,13 @@ class TestTranslationPreview: preview_service = TranslationPreview(db, config_manager, "test-user") with pytest.raises(ValueError, match="source datasource"): - preview_service.preview_rows(job_id="job-123") + await preview_service.preview_rows(job_id="job-123") # endregion test_preview_missing_datasource # region test_preview_missing_translation_column [TYPE Function] # @PURPOSE: Verify error when job has no translation column. - def test_preview_missing_translation_column(self): + @pytest.mark.asyncio + async def test_preview_missing_translation_column(self): """Preview should fail if job has no translation column.""" db = MagicMock() config_manager = MagicMock() @@ -533,7 +538,7 @@ class TestTranslationPreview: preview_service = TranslationPreview(db, config_manager, "test-user") with pytest.raises(ValueError, match="translation column"): - preview_service.preview_rows(job_id="job-123") + await preview_service.preview_rows(job_id="job-123") # endregion test_preview_missing_translation_column diff --git a/backend/src/plugins/translate/_batch_insert.py b/backend/src/plugins/translate/_batch_insert.py index 09afb989..ecefae13 100644 --- a/backend/src/plugins/translate/_batch_insert.py +++ b/backend/src/plugins/translate/_batch_insert.py @@ -1,4 +1,5 @@ # #region BatchInsertService [C:3] [TYPE Module] [SEMANTICS translate, insert, sqllab, upsert, target] +# @defgroup Translate Module group. # @BRIEF Insert successful translation records into target table via Superset SQL Lab. # Builds INSERT SQL (original + per-language rows), resolves backend dialect, # and executes via SupersetSqlLabExecutor. @@ -24,6 +25,7 @@ from .superset_executor import SupersetSqlLabExecutor # #region insert_batch_to_target [C:3] [TYPE Function] [SEMANTICS translate, insert, orchestrate] +# @ingroup Translate # @BRIEF Insert successful records from a single batch into the target table. async def insert_batch_to_target( db: Session, config_manager: ConfigManager, diff --git a/backend/src/plugins/translate/_batch_proc.py b/backend/src/plugins/translate/_batch_proc.py index 19011137..e36a2b80 100644 --- a/backend/src/plugins/translate/_batch_proc.py +++ b/backend/src/plugins/translate/_batch_proc.py @@ -1,4 +1,5 @@ # #region BatchProcessingService [C:4] [TYPE Module] [SEMANTICS translate, batch, process, classify, cache] +# @defgroup Translate Module group. # @BRIEF Batch processing for translation: classify rows (same-language/cache/preview/LLM), # call LLM service, persist TranslationRecord/TranslationLanguage rows. # Local language detection (lingua) replaces LLM-based detection. @@ -32,6 +33,7 @@ from .dictionary import DictionaryManager # #region BatchProcessingService [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Create batch records, classify rows, process LLM calls, persist results. class BatchProcessingService: """Process a batch: classify (cache/preview/LLM), persist, and insert to target.""" @@ -42,6 +44,7 @@ class BatchProcessingService: self._llm_service = LLMTranslationService(db) # #region process_batch [C:3] [TYPE Function] +# @ingroup Translate # @BRIEF Process a single batch: create record, classify rows, call LLM, persist. # @PRE job and batch_rows are valid. # @POST TranslationBatch and TranslationRecord rows are created. diff --git a/backend/src/plugins/translate/_batch_sizer.py b/backend/src/plugins/translate/_batch_sizer.py index e682052f..13b6abbb 100644 --- a/backend/src/plugins/translate/_batch_sizer.py +++ b/backend/src/plugins/translate/_batch_sizer.py @@ -1,4 +1,5 @@ # #region AdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS translate, batch, sizing, token-budget] +# @defgroup Translate Module group. # @BRIEF Adaptive batch sizing for LLM translation — splits source rows into variable-sized # batches based on actual content length and token budget estimates. # Output budget is the PRIMARY constraint; input budget is secondary. @@ -29,6 +30,7 @@ from ._utils import estimate_row_tokens # #region AdaptiveBatchSizer [C:3] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Split source rows into auto-sized batches based on token budget estimates. class AdaptiveBatchSizer: def __init__(self, db: Session, config_manager: ConfigManager) -> None: @@ -36,6 +38,7 @@ class AdaptiveBatchSizer: self.config_manager = config_manager # #region resolve_provider_config [C:2] [TYPE Function] [SEMANTICS llm, provider, model, token-config] +# @ingroup Translate # @BRIEF Resolve provider token config (model name + token limits) for budget estimation. # DB values (context_window, max_output_tokens) take priority over PROVIDER_DEFAULTS. # @POST Returns dict with model, context_window, max_output_tokens. None values = use fallback. @@ -52,6 +55,7 @@ class AdaptiveBatchSizer: # #endregion resolve_provider_config # #region auto_size_batches [C:3] [TYPE Function] [SEMANTICS translate, batch, sizing] +# @ingroup Translate # @BRIEF Split source rows into variable-sized batches based on content length. # Output budget (max_rows_by_output) is the PRIMARY row-count constraint. # @PRE source_rows is non-empty. job has valid config. diff --git a/backend/src/plugins/translate/_lang_detect.py b/backend/src/plugins/translate/_lang_detect.py index 1760680b..079883eb 100644 --- a/backend/src/plugins/translate/_lang_detect.py +++ b/backend/src/plugins/translate/_lang_detect.py @@ -1,4 +1,5 @@ # #region LanguageDetectService [C:2] [TYPE Module] [SEMANTICS translate, language, detection, heuristic] +# @defgroup Translate Module group. # @BRIEF Local language detection powered by lingua-language-detector (no LLM). # Provides fast, non-LLM language identification for source text rows, # reducing LLM token costs by pre-detecting same-language rows and @@ -64,6 +65,7 @@ def _detector_cache_key(target_languages: list[str] | None) -> str: # #region build_detector [C:2] [TYPE Function] [SEMANTICS language, detection, builder] +# @ingroup Translate # @BRIEF Build a LanguageDetector restricted to target + common source languages. # Restricting the language set improves detection speed (~5x faster than # all 87 languages). The detector includes both target languages (for @@ -95,6 +97,7 @@ def build_detector(target_languages: list[str] | None = None) -> LanguageDetecto # #region get_detector [C:2] [TYPE Function] [SEMANTICS language, detection, cache] +# @ingroup Translate # @BRIEF Get or create a cached detector for the given target languages. # Cache keyed by sorted set of language codes — same key returns same instance. def get_detector(target_languages: list[str] | None = None) -> LanguageDetector: @@ -109,6 +112,7 @@ def get_detector(target_languages: list[str] | None = None) -> LanguageDetector: # #region detect_language [C:2] [TYPE Function] [SEMANTICS language, detection] +# @ingroup Translate # @BRIEF Detect the language of a single text string. Returns BCP-47 code or "und". # @RAISES Does not raise — all exceptions caught internally and return "und". def detect_language(text: str, detector: LanguageDetector) -> str: @@ -178,6 +182,7 @@ def _character_block_fallback( # #region batch_detect [C:3] [TYPE Function] [SEMANTICS language, detection, batch] +# @ingroup Translate # @BRIEF Detect language for multiple texts in batch. Builds detector if not provided. # @RELATION DEPENDS_ON -> [detect_language] # @RELATION DEPENDS_ON -> [get_detector] diff --git a/backend/src/plugins/translate/_llm_async_http.py b/backend/src/plugins/translate/_llm_async_http.py index 70d43a7e..e5698e39 100644 --- a/backend/src/plugins/translate/_llm_async_http.py +++ b/backend/src/plugins/translate/_llm_async_http.py @@ -1,4 +1,5 @@ # #region LLMAsyncHttpClient [C:4] [TYPE Module] [SEMANTICS translate, llm, http, openai, async, retry, rate-limit] +# @defgroup Translate Module group. # @BRIEF Async HTTP client for OpenAI-compatible LLM API calls using httpx.AsyncClient # with rate-limit handling and structured output fallback. # @LAYER Infrastructure @@ -61,6 +62,7 @@ async def _get_http_client() -> httpx.AsyncClient: # #region call_openai_compatible [C:3] [TYPE Function] [SEMANTICS translate, llm, http, openai, async] +# @ingroup Translate # @BRIEF Call OpenAI-compatible API asynchronously with rate-limit handling and structured output fallback. # @PRE Valid API endpoint, key, model, and prompt. # @POST Returns (response text, finish_reason). diff --git a/backend/src/plugins/translate/_llm_call.py b/backend/src/plugins/translate/_llm_call.py index d474db37..f07ccd6c 100644 --- a/backend/src/plugins/translate/_llm_call.py +++ b/backend/src/plugins/translate/_llm_call.py @@ -1,4 +1,5 @@ # #region LLMTranslationService [C:4] [TYPE Module] [SEMANTICS translate, llm, call, orchestrate, retry] +# @defgroup Translate Module group. # @BRIEF LLM interaction for batch translation: call provider with retry, handle truncation # by recursive splitting, enforce dictionary post-processing. Orchestrates HTTP calls # (_llm_http) and response parsing (_llm_parse). @@ -38,6 +39,7 @@ MAX_RETRIES_PER_BATCH = 3 # #region LLMTranslationService [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Call LLM, handle retry/truncation, parse response, persist records. class LLMTranslationService: @@ -45,6 +47,7 @@ class LLMTranslationService: self.db = db # #region call_llm_for_batch [C:3] [TYPE Function] [SEMANTICS translate, llm, batch, orchestrate] +# @ingroup Translate # @BRIEF Call LLM for a batch of rows requiring translation. Parse and persist results. # @PRE job has valid provider_id. batch_rows is non-empty. # @POST Returns dict with successful/failed/skipped counts. @@ -484,6 +487,7 @@ class LLMTranslationService: # #endregion _add_skipped # #region call_llm [C:3] [TYPE Function] [SEMANTICS translate, llm, call] +# @ingroup Translate # @BRIEF Route to provider-specific LLM call implementation. async def call_llm(self, job: TranslationJob, prompt: str, max_tokens: int = 8192) -> tuple[str, str | None]: with belief_scope("LLMTranslationService.call_llm"): diff --git a/backend/src/plugins/translate/_llm_parse.py b/backend/src/plugins/translate/_llm_parse.py index 069f4c40..d4316919 100644 --- a/backend/src/plugins/translate/_llm_parse.py +++ b/backend/src/plugins/translate/_llm_parse.py @@ -1,4 +1,5 @@ # #region LLMResponseParser [C:3] [TYPE Module] [SEMANTICS translate, llm, parse, json, recovery] +# @defgroup Translate Module group. # @BRIEF Parse LLM JSON response into per-row translations with support for markdown code # blocks, truncated JSON recovery, and multi-language format. # @LAYER Domain @@ -13,6 +14,7 @@ from ...core.logger import logger # #region parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate, llm, parse, json] +# @ingroup Translate # @BRIEF Parse LLM JSON response into dict of row_id -> per-language translations. # @PRE response_text is valid JSON with {"rows": [...]} structure. # @POST Returns dict mapping row_id to dict with 'detected_source_language' and per-language codes. diff --git a/backend/src/plugins/translate/_run_service.py b/backend/src/plugins/translate/_run_service.py index 09306c67..316c3a28 100644 --- a/backend/src/plugins/translate/_run_service.py +++ b/backend/src/plugins/translate/_run_service.py @@ -1,4 +1,5 @@ # #region RunExecutionService [C:4] [TYPE Module] [SEMANTICS translate, run, execution, orchestration] +# @defgroup Translate Module group. # @BRIEF Full run lifecycle: prepare run, fetch source rows, filter new keys, orchestrate batches, # handle cancellation, update per-language stats, finalize run status. # @LAYER Domain @@ -23,6 +24,7 @@ from ._run_source import _extract_chart_data_rows, fetch_source_rows # #region RunExecutionService [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Orchestrate full translation run: fetch data, process batches, finalize. class RunExecutionService: """Orchestrate a full translation run: prepare, process batches, finalize.""" diff --git a/backend/src/plugins/translate/_run_source.py b/backend/src/plugins/translate/_run_source.py index c6a532fc..d900f4a2 100644 --- a/backend/src/plugins/translate/_run_source.py +++ b/backend/src/plugins/translate/_run_source.py @@ -1,4 +1,5 @@ # #region RunSourceFetcher [C:3] [TYPE Module] [SEMANTICS translate, source, fetch, superset, preview] +# @defgroup Translate Module group. # @BRIEF Fetch source rows for translation runs from Superset datasource or preview session. # Extracted from RunExecutionService to comply with INV_7 (< 400 lines/module). # @LAYER Domain @@ -24,6 +25,7 @@ MAX_ROWS_PER_RUN = 10000 # #region fetch_source_rows [C:3] [TYPE Function] [SEMANTICS translate, source, fetch] +# @ingroup Translate # @BRIEF Fetch source rows from Superset datasource or preview session fallback. async def fetch_source_rows(db: Session, config_manager: ConfigManager, job_id: str, run_id: str) -> list[dict[str, Any]]: """Fetch source rows from Superset datasource or preview session fallback.""" diff --git a/backend/src/plugins/translate/_text_cleaner.py b/backend/src/plugins/translate/_text_cleaner.py index 25922182..3be27fb4 100644 --- a/backend/src/plugins/translate/_text_cleaner.py +++ b/backend/src/plugins/translate/_text_cleaner.py @@ -1,4 +1,5 @@ # #region TextCleaner [C:3] [TYPE Module] [SEMANTICS translate, text, cleaning, whitespace, truncation] +# @defgroup Translate Module group. # @BRIEF Text cleaning utilities for the translation pipeline — whitespace normalization # and character-length truncation. Used as a preprocessing step before LLM calls. # @LAYER Domain @@ -8,6 +9,7 @@ # Normalizing whitespace before truncation ensures accurate character counts. # #region normalize_whitespace [C:2] [TYPE Function] [SEMANTICS text, whitespace, normalize] +# @ingroup Translate # @BRIEF Collapse multiple spaces, tabs, and newlines into single spaces; trim leading/trailing # whitespace. Returns empty string for empty or whitespace-only input. # @PRE text is a string (empty string allowed). @@ -27,6 +29,7 @@ def normalize_whitespace(text: str) -> str: # #region truncate_text [C:2] [TYPE Function] [SEMANTICS text, truncation, length] +# @ingroup Translate # @BRIEF Truncate text to max_length characters, appending "..." if truncation occurs. # @PRE text is a string. max_length >= 0. # @POST When len(text) <= max_length, returns text unchanged. @@ -46,6 +49,7 @@ def truncate_text(text: str, max_length: int = 500) -> str: # #region clean_text [C:2] [TYPE Function] [SEMANTICS text, clean, normalize, truncate] +# @ingroup Translate # @BRIEF Combine whitespace normalization and truncation into one step. # Applies normalize_whitespace first, then truncate_text. # @PRE text is a string. max_length >= 0. diff --git a/backend/src/plugins/translate/_token_budget.py b/backend/src/plugins/translate/_token_budget.py index 5865595f..efccdcf8 100644 --- a/backend/src/plugins/translate/_token_budget.py +++ b/backend/src/plugins/translate/_token_budget.py @@ -1,4 +1,5 @@ # #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm] +# @defgroup Translate Module group. # @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits. # Output budget is now the PRIMARY constraint — finish_reason=length is usually output exceeding max_tokens, not input overflowing context_window. # @LAYER Domain @@ -14,20 +15,24 @@ # Input-only batch sizing — output budget is the primary truncation cause (finish_reason=length). # #region DEFAULT_CONTEXT_WINDOW [TYPE Constant] +# @ingroup Translate DEFAULT_CONTEXT_WINDOW = 64000 # #endregion DEFAULT_CONTEXT_WINDOW # #region DEFAULT_MAX_OUTPUT_TOKENS [TYPE Constant] +# @ingroup Translate DEFAULT_MAX_OUTPUT_TOKENS = 16384 # #endregion DEFAULT_MAX_OUTPUT_TOKENS # @BRIEF CoT reasoning overhead tokens for DeepSeek models (~2000 tokens for chain-of-thought). # #region REASONING_OVERHEAD [TYPE Constant] +# @ingroup Translate REASONING_OVERHEAD = 2000 # #endregion REASONING_OVERHEAD # #region PROVIDER_DEFAULTS [TYPE Constant] +# @ingroup Translate # Maps model name (or "default" fallback) to capacity limits. # @RATIONALE Different providers have drastically different context windows and @@ -44,35 +49,42 @@ PROVIDER_DEFAULTS: dict[str, dict[str, int]] = { # #endregion PROVIDER_DEFAULTS # Increased from 60 to 120 because SQL/dashboard text and JSON structure need more. # #region OUTPUT_PER_ROW_PER_LANG [TYPE Constant] +# @ingroup Translate # Increased from 60 to 120 because SQL/dashboard text and JSON structure need more. OUTPUT_PER_ROW_PER_LANG = 120 # #endregion OUTPUT_PER_ROW_PER_LANG # #region JSON_OVERHEAD_PER_ROW [TYPE Constant] +# @ingroup Translate JSON_OVERHEAD_PER_ROW = 50 # #endregion JSON_OVERHEAD_PER_ROW # #region PROMPT_BASE_TOKENS [TYPE Constant] +# @ingroup Translate # Increased from 300 to 600 to account for longer template, system msg, and dict preamble. PROMPT_BASE_TOKENS = 600 # #endregion PROMPT_BASE_TOKENS # @BRIEF Cap for dictionary tokens in estimation to avoid overestimating. # #region DICT_TOKENS_PER_ENTRY [TYPE Constant] +# @ingroup Translate DICT_TOKENS_PER_ENTRY = 20 # #endregion DICT_TOKENS_PER_ENTRY # #region DICT_TOKENS_MAX [TYPE Constant] +# @ingroup Translate DICT_TOKENS_MAX = 5000 # #endregion DICT_TOKENS_MAX # #region CHARS_PER_TOKEN_MIXED [TYPE Constant] +# @ingroup Translate # @BRIEF Deprecated — use CJK_RATIO + OTHER_RATIO instead. Kept for backward compat in estimate_row_tokens. CHARS_PER_TOKEN_MIXED = 2.2 # #endregion CHARS_PER_TOKEN_MIXED # #region CJK_RATIO [TYPE Constant] +# @ingroup Translate # @BRIEF Conservative CJK chars/token ratio. Modern models (Qwen, DeepSeek) tokenize at ~1.0-1.3. # @RATIONALE Lowered from 1.5 to 1.0 to produce more conservative (higher) token estimates. # Actual token density varies by model tokenizer; this is intentionally pessimistic. @@ -80,11 +92,13 @@ CJK_RATIO = 1.0 # #endregion CJK_RATIO # #region OTHER_RATIO [TYPE Constant] +# @ingroup Translate # @BRIEF Conservative ratio for non-CJK text. cl100k_base averages ~1.8-2.5. OTHER_RATIO = 1.8 # #endregion OTHER_RATIO # #region INPUT_SAFETY_FACTOR [TYPE Constant] +# @ingroup Translate # @BRIEF Multiplier applied to per-batch input budget in batch sizer. # Accounts for CJK estimation variance across different tokenizers. # @RATIONALE Single centralised factor instead of scattered margins in multiple functions. @@ -93,15 +107,18 @@ INPUT_SAFETY_FACTOR = 0.75 # #endregion INPUT_SAFETY_FACTOR # #region OUTPUT_SAFETY_FACTOR [TYPE Constant] +# @ingroup Translate # @BRIEF Multiplier applied to output budget when computing max rows per batch. OUTPUT_SAFETY_FACTOR = 0.70 # #endregion OUTPUT_SAFETY_FACTOR # #region MIN_MAX_TOKENS [TYPE Constant] +# @ingroup Translate MIN_MAX_TOKENS = 4096 # #endregion MIN_MAX_TOKENS # #region MAX_OUTPUT_HEADROOM [TYPE Constant] +# @ingroup Translate # Increased from 1000 to 3000 because SQL/dashboard text output varies significantly. MAX_OUTPUT_HEADROOM = 3000 diff --git a/backend/src/plugins/translate/_utils.py b/backend/src/plugins/translate/_utils.py index e9d7529e..7221006b 100644 --- a/backend/src/plugins/translate/_utils.py +++ b/backend/src/plugins/translate/_utils.py @@ -1,4 +1,5 @@ # #region TranslationUtils [C:3] [TYPE Module] [SEMANTICS translate, utils, hash, dictionary, cache] +# @defgroup Translate Module group. # @BRIEF Shared utility functions for the translation plugin — dictionary enforcement, # source hashing, cache lookup. Extracted from executor.py to break circular imports. # @LAYER Domain @@ -197,6 +198,7 @@ def _compute_key_hash(source_data: dict) -> str: # #region estimate_row_tokens [C:2] [TYPE Function] [SEMANTICS translate, token, estimation] +# @ingroup Translate # @BRIEF Estimate token count for a single source row including context fields. # @PRE source_text is a string. # @POST Returns estimated token count >= 1. diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index b6321a1c..c2d59569 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -1,4 +1,5 @@ # #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, dictionary, batch, term] +# @defgroup Translate Module group. # @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationJob] @@ -22,6 +23,7 @@ from .dictionary_import_export import DictionaryImportExport # #region DictionaryManager [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Facade for terminology dictionaries: delegates to DictionaryCRUD, DictionaryEntryCRUD, DictionaryImportExport, DictionaryBatchFilter, DictionaryCorrectionService. # @PRE Database session is open and valid. # @POST Dictionary and entry mutations are persisted with conflict detection. diff --git a/backend/src/plugins/translate/dictionary_correction.py b/backend/src/plugins/translate/dictionary_correction.py index 005cd9ce..a15a766a 100644 --- a/backend/src/plugins/translate/dictionary_correction.py +++ b/backend/src/plugins/translate/dictionary_correction.py @@ -1,4 +1,5 @@ # #region DictionaryCorrectionService [C:3] [TYPE Class] [SEMANTICS dictionary, correction, bulk] +# @defgroup Translate Module group. # @BRIEF Submit term corrections and bulk corrections to dictionaries with conflict detection. # @RELATION DEPENDS_ON -> [DictionaryEntry] # @RELATION DEPENDS_ON -> [TerminologyDictionary] diff --git a/backend/src/plugins/translate/dictionary_crud.py b/backend/src/plugins/translate/dictionary_crud.py index f8170ae1..8cc466bf 100644 --- a/backend/src/plugins/translate/dictionary_crud.py +++ b/backend/src/plugins/translate/dictionary_crud.py @@ -1,4 +1,5 @@ # #region DictionaryCRUD [C:3] [TYPE Class] [SEMANTICS dictionary, crud, terminology] +# @defgroup Translate Module group. # @BRIEF CRUD operations for TerminologyDictionary records. # @RELATION DEPENDS_ON -> [TerminologyDictionary] # @RELATION DEPENDS_ON -> [TranslationJob] diff --git a/backend/src/plugins/translate/dictionary_entries.py b/backend/src/plugins/translate/dictionary_entries.py index 72a423e9..0f1ff173 100644 --- a/backend/src/plugins/translate/dictionary_entries.py +++ b/backend/src/plugins/translate/dictionary_entries.py @@ -1,4 +1,5 @@ # #region DictionaryEntryCRUD [C:3] [TYPE Class] [SEMANTICS dictionary, entries, crud] +# @defgroup Translate Module group. # @BRIEF CRUD operations for DictionaryEntry records. # @RELATION DEPENDS_ON -> [DictionaryEntry] # @RELATION DEPENDS_ON -> [TerminologyDictionary] diff --git a/backend/src/plugins/translate/dictionary_filter.py b/backend/src/plugins/translate/dictionary_filter.py index ff05c781..0db50238 100644 --- a/backend/src/plugins/translate/dictionary_filter.py +++ b/backend/src/plugins/translate/dictionary_filter.py @@ -1,4 +1,5 @@ # #region DictionaryBatchFilter [C:3] [TYPE Class] [SEMANTICS dictionary, filter, batch, matching] +# @defgroup Translate Module group. # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against dictionaries. # @RELATION DEPENDS_ON -> [DictionaryEntry] # @RELATION DEPENDS_ON -> [TranslationJobDictionary] diff --git a/backend/src/plugins/translate/dictionary_import_export.py b/backend/src/plugins/translate/dictionary_import_export.py index e533fa12..d091353d 100644 --- a/backend/src/plugins/translate/dictionary_import_export.py +++ b/backend/src/plugins/translate/dictionary_import_export.py @@ -1,4 +1,5 @@ # #region DictionaryImportExport [C:3] [TYPE Class] [SEMANTICS dictionary, import, export, csv, migration] +# @defgroup Translate Module group. # @BRIEF Import/export entries as CSV/TSV with conflict detection, and migration of old entries. # @RELATION DEPENDS_ON -> [DictionaryEntry] # @RELATION DEPENDS_ON -> [TerminologyDictionary] diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index cbf0b1ef..873c32b1 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -1,4 +1,5 @@ # #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS sqlalchemy, translate, event, log, audit] +# @defgroup Translate Module group. # @BRIEF Structured event logging for translation operations with terminal event invariant enforcement. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationEvent] @@ -38,6 +39,7 @@ DEFAULT_RETENTION_DAYS = 90 # #region TranslationEventLog [C:5] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Structured event logging for translation operations with terminal event invariant enforcement. # @PRE Database session is available. # @POST Events are written immutably; terminal events enforced per run. diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index 611ec612..fec5975f 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -1,4 +1,5 @@ # #region TranslationExecutor [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, orchestration] +# @defgroup Translate Module group. # @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch # and TranslationRecord rows. Thin orchestrator — delegates to focused sub-services. # @LAYER Domain diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index 621d8ed3..707f421c 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -1,4 +1,5 @@ # #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, metrics, job, statistics] +# @defgroup Translate Module group. # @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationSchedule] @@ -23,6 +24,7 @@ from ...models.translate import ( # #region TranslationMetrics [C:3] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Aggregate translation metrics from live events and MetricSnapshot. class TranslationMetrics: diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index feb03cf9..1ac303ef 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -1,4 +1,5 @@ # #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS sqlalchemy, tenacity, translate, orchestration, batch] +# @defgroup Translate Module group. # @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationRun] @@ -34,6 +35,7 @@ from .orchestrator_runner import TranslationStageRunner # #region TranslationOrchestrator [C:5] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. # @PRE DB session and config manager are available. # @POST Runs are created, executed, and finalized with event records. diff --git a/backend/src/plugins/translate/orchestrator_aggregator.py b/backend/src/plugins/translate/orchestrator_aggregator.py index df8096a3..a125a69d 100644 --- a/backend/src/plugins/translate/orchestrator_aggregator.py +++ b/backend/src/plugins/translate/orchestrator_aggregator.py @@ -1,4 +1,5 @@ # #region TranslationResultAggregator [C:4] [TYPE Class] [SEMANTICS sqlalchemy, translate, query, history, records] +# @defgroup Translate Module group. # @BRIEF Query translation run status, records, and history. # @PRE Database session is available. # @POST Returns structured run data with pagination. diff --git a/backend/src/plugins/translate/orchestrator_cancel.py b/backend/src/plugins/translate/orchestrator_cancel.py index 231fcce3..bf96d63e 100644 --- a/backend/src/plugins/translate/orchestrator_cancel.py +++ b/backend/src/plugins/translate/orchestrator_cancel.py @@ -1,4 +1,5 @@ # #region orchestrator_cancel [C:3] [TYPE Module] [SEMANTICS translate, cancel, retry_insert] +# @defgroup Translate Module group. # @BRIEF Cancel and retry-insert operations for translation runs. # @RELATION DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [TranslationJob] @@ -37,6 +38,7 @@ def _fallback_cancel_request(db: Session, run_id: str) -> TranslationRun: # #region cancel_run [C:3] [TYPE Function] [SEMANTICS translate, cancel, run] +# @ingroup Translate # @BRIEF Cancel a running translation run. # @SIDE_EFFECT DB writes; event log. # @RELATION DEPENDS_ON -> [TranslationRun] @@ -80,6 +82,7 @@ def cancel_run( # #region retry_insert [C:3] [TYPE Function] [SEMANTICS translate, retry, insert] +# @ingroup Translate # @BRIEF Retry the SQL insert phase for a completed run. # @SIDE_EFFECT Superset API call; DB writes. # @RELATION DEPENDS_ON -> [TranslationRun] diff --git a/backend/src/plugins/translate/orchestrator_config.py b/backend/src/plugins/translate/orchestrator_config.py index 01e81bc8..eeef65b6 100644 --- a/backend/src/plugins/translate/orchestrator_config.py +++ b/backend/src/plugins/translate/orchestrator_config.py @@ -1,4 +1,5 @@ # #region orchestrator_config [C:3] [TYPE Module] [SEMANTICS hash, config, snapshot, translate] +# @defgroup Translate Module group. # @BRIEF Config hash and dictionary snapshot hash utilities for translation planning. # @RELATION DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [TranslationJobDictionary] @@ -8,6 +9,7 @@ import json # #region compute_config_hash [C:2] [TYPE Function] [SEMANTICS config, hash, deterministic] +# @ingroup Translate # @BRIEF Compute a deterministic hash of job configuration for snapshot comparison. # @PRE job has valid configuration attributes. # @POST Returns 16-char hex SHA-256 hash. @@ -29,6 +31,7 @@ def compute_config_hash(job) -> str: # #region compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS dictionary, hash, snapshot] +# @ingroup Translate # @BRIEF Compute a hash of dictionary state for snapshot comparison. # Includes dictionary IDs, entry count, and max entry updated_at # to invalidate cache when dictionary entries are modified. diff --git a/backend/src/plugins/translate/orchestrator_exec.py b/backend/src/plugins/translate/orchestrator_exec.py index 721ec34d..451a41a1 100644 --- a/backend/src/plugins/translate/orchestrator_exec.py +++ b/backend/src/plugins/translate/orchestrator_exec.py @@ -1,4 +1,5 @@ # #region TranslationExecutionEngine [C:4] [TYPE Class] [SEMANTICS translate, execution, run, orchestration] +# @defgroup Translate Module group. # @BRIEF Execute translation runs: dispatch executor, manage completion and failure paths. # @PRE Database session and config manager are available. Run is in valid state. # @POST Run is executed with SQL generated; events recorded. diff --git a/backend/src/plugins/translate/orchestrator_lang_stats.py b/backend/src/plugins/translate/orchestrator_lang_stats.py index 2f74ee9b..5fbba1d9 100644 --- a/backend/src/plugins/translate/orchestrator_lang_stats.py +++ b/backend/src/plugins/translate/orchestrator_lang_stats.py @@ -1,4 +1,5 @@ # #region update_language_stats [C:3] [TYPE Function] [SEMANTICS translate, language, stats, aggregation] +# @ingroup Translate # @BRIEF Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats. # @SIDE_EFFECT DB writes on language_stats objects. # @RELATION DEPENDS_ON -> [TranslationRecord] diff --git a/backend/src/plugins/translate/orchestrator_planner.py b/backend/src/plugins/translate/orchestrator_planner.py index d5ce3188..e939cfb1 100644 --- a/backend/src/plugins/translate/orchestrator_planner.py +++ b/backend/src/plugins/translate/orchestrator_planner.py @@ -1,4 +1,5 @@ # #region TranslationPlanner [C:4] [TYPE Class] [SEMANTICS sqlalchemy, translate, orchestration, planning] +# @defgroup Translate Module group. # @BRIEF Handle translation run planning: validation, config hashing, dictionary snapshot. # @PRE Database session and config manager are available. # @POST TranslationRun is planned with hashes and config snapshot; events recorded. diff --git a/backend/src/plugins/translate/orchestrator_query.py b/backend/src/plugins/translate/orchestrator_query.py index a3b5e4f1..ba4ef271 100644 --- a/backend/src/plugins/translate/orchestrator_query.py +++ b/backend/src/plugins/translate/orchestrator_query.py @@ -1,4 +1,5 @@ # #region orchestrator_query [C:3] [TYPE Module] [SEMANTICS translate, query, records, history] +# @defgroup Translate Module group. # @BRIEF Query translation run records and history with pagination. # @RELATION DEPENDS_ON -> [TranslationRecord] # @RELATION DEPENDS_ON -> [TranslationRun] @@ -11,6 +12,7 @@ from ...models.translate import TranslationRecord, TranslationRun # #region get_run_records [C:2] [TYPE Function] [SEMANTICS records, query, paginated] +# @ingroup Translate # @BRIEF Get paginated records for a run. Supports deduplicate=true to return unique source_hash rows. def get_run_records( db: Session, @@ -106,6 +108,7 @@ def get_run_records( # #endregion get_run_records # #region get_run_history [C:2] [TYPE Function] [SEMANTICS history, query, paginated] +# @ingroup Translate # @BRIEF Get run history for a job with pagination. Returns trigger_type, cache_hits for enhanced run list display. def get_run_history( db: Session, diff --git a/backend/src/plugins/translate/orchestrator_retry.py b/backend/src/plugins/translate/orchestrator_retry.py index 43172582..e2fc8116 100644 --- a/backend/src/plugins/translate/orchestrator_retry.py +++ b/backend/src/plugins/translate/orchestrator_retry.py @@ -1,4 +1,5 @@ # #region TranslationRunRetryManager [C:4] [TYPE Class] [SEMANTICS translate, retry] +# @defgroup Translate Module group. # @BRIEF Manage retry of translation run batches. # @PRE Database session and config manager are available. # @POST Retried batches are re-executed. diff --git a/backend/src/plugins/translate/orchestrator_run_completion.py b/backend/src/plugins/translate/orchestrator_run_completion.py index 6eadd7ba..3e2b04d1 100644 --- a/backend/src/plugins/translate/orchestrator_run_completion.py +++ b/backend/src/plugins/translate/orchestrator_run_completion.py @@ -1,4 +1,5 @@ # #region orchestrator_run_completion [C:3] [TYPE Module] [SEMANTICS translate, run, completion, failure] +# @defgroup Translate Module group. # @BRIEF Post-execution run completion handlers: cancelled, success-with-insert, and failure paths. # @RELATION DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [TranslationJob] diff --git a/backend/src/plugins/translate/orchestrator_runner.py b/backend/src/plugins/translate/orchestrator_runner.py index a0c611b4..5e04f150 100644 --- a/backend/src/plugins/translate/orchestrator_runner.py +++ b/backend/src/plugins/translate/orchestrator_runner.py @@ -1,4 +1,5 @@ # #region TranslationStageRunner [C:4] [TYPE Class] [SEMANTICS sqlalchemy, translate, execution, superset] +# @defgroup Translate Module group. # @BRIEF Coordinate translation run execution, retry, and cancellation via delegated components. # @PRE Database session and config manager are available. Run is in valid state. # @POST Run is executed; events recorded. diff --git a/backend/src/plugins/translate/orchestrator_sql.py b/backend/src/plugins/translate/orchestrator_sql.py index 366ec152..12462bca 100644 --- a/backend/src/plugins/translate/orchestrator_sql.py +++ b/backend/src/plugins/translate/orchestrator_sql.py @@ -1,4 +1,5 @@ # #region SQLInsertService [C:4] [TYPE Class] [SEMANTICS sql, insert, superset, translate] +# @defgroup Translate Module group. # @BRIEF Generate INSERT SQL from translation records and submit to Superset SQL Lab. # @PRE Job has target table configured. Run has successful records. # @POST SQL is generated and submitted to Superset; returns execution result. diff --git a/backend/src/plugins/translate/orchestrator_sql_rows.py b/backend/src/plugins/translate/orchestrator_sql_rows.py index f064daeb..f685f311 100644 --- a/backend/src/plugins/translate/orchestrator_sql_rows.py +++ b/backend/src/plugins/translate/orchestrator_sql_rows.py @@ -1,4 +1,5 @@ # #region orchestrator_sql_rows [C:3] [TYPE Module] [SEMANTICS sql, insert, row, column, builders] +# @defgroup Translate Module group. # @BRIEF Column and row building utilities for SQL insertion in translate plugin. # @RELATION DEPENDS_ON -> [TranslationRecord] # @RELATION DEPENDS_ON -> [TranslationJob] @@ -10,6 +11,7 @@ from .sql_generator import _normalize_timestamp_value # #region build_columns [C:2] [TYPE Function] [SEMANTICS sql, columns, build] +# @ingroup Translate # @BRIEF Build list of target columns for SQL INSERT from job configuration. def build_columns(job: TranslationJob, effective_target: str | None) -> list[str]: """Build column list for SQL INSERT from job config.""" @@ -48,6 +50,7 @@ def build_context_keys(job: TranslationJob, effective_target: str | None) -> lis # #region build_rows [C:2] [TYPE Function] [SEMANTICS sql, rows, build] +# @ingroup Translate # @BRIEF Build row data for SQL INSERT with per-language expansion. # @SIDE_EFFECT Reads translation record language entries. def build_rows( diff --git a/backend/src/plugins/translate/orchestrator_validation.py b/backend/src/plugins/translate/orchestrator_validation.py index 9f859115..f845ffca 100644 --- a/backend/src/plugins/translate/orchestrator_validation.py +++ b/backend/src/plugins/translate/orchestrator_validation.py @@ -1,4 +1,5 @@ # #region validate_job_preconditions [C:3] [TYPE Function] [SEMANTICS validation, preconditions, translate] +# @ingroup Translate # @BRIEF Validate preconditions before starting a translation run. # @PRE Job exists and db session is valid. # @POST Raises ValueError if any precondition fails. diff --git a/backend/src/plugins/translate/plugin.py b/backend/src/plugins/translate/plugin.py index 1099c537..1cb00ad5 100644 --- a/backend/src/plugins/translate/plugin.py +++ b/backend/src/plugins/translate/plugin.py @@ -1,4 +1,5 @@ # #region TranslatePlugin [C:2] [TYPE Module] [SEMANTICS clickhouse, translate, sql, dashboard, dialect] +# @defgroup Translate Module group. # @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation. # @LAYER Domain # @RELATION INHERITS -> [PluginBase] @@ -11,6 +12,7 @@ from ...core.plugin_base import PluginBase # #region TranslatePlugin [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects. # @RELATION IMPLEMENTS -> [PluginBase] class TranslatePlugin(PluginBase): diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index 10141670..6cb91f54 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -1,4 +1,5 @@ # #region TranslationPreview [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, preview, llm, review] +# @defgroup Translate Module group. # @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationJob] @@ -43,6 +44,7 @@ from .preview_review import PreviewSessionManager # #region TranslationPreview [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate. # @PRE Database session and config manager are available. # @POST Preview sessions created with persisted records; full execution gates on accepted session. @@ -131,7 +133,7 @@ class TranslationPreview: llm_response = await self._executor.call_llm( job=job, prompt=prompt_data["prompt"], max_tokens=token_budget["max_output_needed"], ) - logger.reason(f"TIMING: LLM call: {time.monotonic() - t_llm:.2f}s", {}) + logger.reason(f"TIMING: LLM call: {time.monotonic() - t_llm:.2f}s") translations = self._executor.parse_llm_response( llm_response, prompt_data["actual_row_count"], target_languages=target_languages, diff --git a/backend/src/plugins/translate/preview_executor.py b/backend/src/plugins/translate/preview_executor.py index 5f978d95..e2033e37 100644 --- a/backend/src/plugins/translate/preview_executor.py +++ b/backend/src/plugins/translate/preview_executor.py @@ -1,4 +1,5 @@ # #region PreviewExecutor [C:4] [TYPE Class] [SEMANTICS sqlalchemy, superset, llm, preview] +# @defgroup Translate Module group. # @BRIEF Fetch sample data from Superset and call LLM provider for preview. # @PRE Database session, config manager, and Superset client are available. # @POST Sample rows fetched, LLM called. diff --git a/backend/src/plugins/translate/preview_prompt_builder.py b/backend/src/plugins/translate/preview_prompt_builder.py index 03c962a8..6eb349ce 100644 --- a/backend/src/plugins/translate/preview_prompt_builder.py +++ b/backend/src/plugins/translate/preview_prompt_builder.py @@ -1,4 +1,5 @@ # #region PreviewPromptBuilder [C:3] [TYPE Class] [SEMANTICS prompt, dictionary, preview] +# @defgroup Translate Module group. # @BRIEF Build LLM prompts for preview sessions including dictionary glossary and row context. # @RELATION DEPENDS_ON -> [DictionaryManager] # @RELATION DEPENDS_ON -> [render_prompt] diff --git a/backend/src/plugins/translate/preview_prompt_helpers.py b/backend/src/plugins/translate/preview_prompt_helpers.py index aebe20b3..702b35d0 100644 --- a/backend/src/plugins/translate/preview_prompt_helpers.py +++ b/backend/src/plugins/translate/preview_prompt_helpers.py @@ -1,4 +1,5 @@ # #region preview_prompt_helpers [C:2] [TYPE Module] [SEMANTICS token, budget, metadata, estimate] +# @defgroup Translate Module group. # @BRIEF Token estimation metadata and budget helpers for preview prompt building. # @RELATION DEPENDS_ON -> [estimate_token_budget] # @RELATION DEPENDS_ON -> [TokenEstimator] @@ -12,6 +13,7 @@ from .preview_token_estimator import TokenEstimator # #region estimate_token_budget_for_rows [C:2] [TYPE Function] [SEMANTICS token, budget, estimate, truncate] +# @ingroup Translate # @BRIEF Check token budget and optionally truncate source rows for preview. # @POST Returns adjusted source_rows, actual_row_count, and token_budget dict. def estimate_token_budget_for_rows( diff --git a/backend/src/plugins/translate/preview_resolve_provider.py b/backend/src/plugins/translate/preview_resolve_provider.py index cc7891b6..a761965c 100644 --- a/backend/src/plugins/translate/preview_resolve_provider.py +++ b/backend/src/plugins/translate/preview_resolve_provider.py @@ -1,4 +1,5 @@ # #region resolve_provider_model [C:2] [TYPE Function] [SEMANTICS llm, provider, model, resolve] +# @ingroup Translate # @BRIEF Resolve the LLM provider model name for a translation job. # @RELATION DEPENDS_ON -> [LLMProviderService] from sqlalchemy.orm import Session diff --git a/backend/src/plugins/translate/preview_response_parser.py b/backend/src/plugins/translate/preview_response_parser.py index 8292d96f..0cfafe62 100644 --- a/backend/src/plugins/translate/preview_response_parser.py +++ b/backend/src/plugins/translate/preview_response_parser.py @@ -1,4 +1,5 @@ # #region preview_response_parser [C:3] [TYPE Module] [SEMANTICS llm, parse, json, superset, hash] +# @defgroup Translate Module group. # @BRIEF Parse LLM JSON responses and Superset data; compute config/dict hashes for preview. # @RELATION DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [TranslationJobDictionary] @@ -14,6 +15,7 @@ from ...models.translate import TranslationJob, TranslationJobDictionary # #region parse_llm_response [C:3] [TYPE Function] [SEMANTICS llm, parse, json, translate] +# @ingroup Translate # @BRIEF Parse LLM JSON response into structured translations dict with per-language support. # @PRE response_text is a valid JSON string (possibly wrapped in markdown code block). # @POST Returns dict mapping row_id -> {detected_source_language, lang_code: translation, ...}. @@ -122,6 +124,7 @@ def compute_config_hash(job: TranslationJob) -> str: # #region compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS dictionary, hash, snapshot] +# @ingroup Translate # @BRIEF Compute a hash of dictionary state for snapshot comparison. # Includes dictionary IDs, entry count, and max entry updated_at. def compute_dict_snapshot_hash(db: Session, job_id: str) -> str: diff --git a/backend/src/plugins/translate/preview_review.py b/backend/src/plugins/translate/preview_review.py index 342eb448..c76da662 100644 --- a/backend/src/plugins/translate/preview_review.py +++ b/backend/src/plugins/translate/preview_review.py @@ -1,4 +1,5 @@ # #region PreviewSessionManager [C:3] [TYPE Class] [SEMANTICS preview, session, review, approve] +# @defgroup Translate Module group. # @BRIEF Manage preview session row-level actions: approve/edit/reject rows. # @RELATION DEPENDS_ON -> [TranslationPreviewSession] # @RELATION DEPENDS_ON -> [TranslationPreviewRecord] diff --git a/backend/src/plugins/translate/preview_session_ops.py b/backend/src/plugins/translate/preview_session_ops.py index 47c28e36..6a14340a 100644 --- a/backend/src/plugins/translate/preview_session_ops.py +++ b/backend/src/plugins/translate/preview_session_ops.py @@ -1,4 +1,5 @@ # #region preview_session_ops [C:3] [TYPE Module] [SEMANTICS preview, session, accept, get] +# @defgroup Translate Module group. # @BRIEF Preview session lifecycle operations: accept and query preview sessions. # @RELATION DEPENDS_ON -> [TranslationPreviewSession] # @RELATION DEPENDS_ON -> [TranslationPreviewRecord] @@ -21,6 +22,7 @@ from .preview_session_serializer import ( # #region accept_preview_session [C:2] [TYPE Function] [SEMANTICS preview, session, accept] +# @ingroup Translate # @BRIEF Mark a preview session as accepted, which gates full execution. # @SIDE_EFFECT DB writes on session status. def accept_preview_session(db: Session, job_id: str, _current_user: str | None = None) -> dict[str, Any]: @@ -60,6 +62,7 @@ def accept_preview_session(db: Session, job_id: str, _current_user: str | None = # #region get_preview_session [C:2] [TYPE Function] [SEMANTICS preview, session, query] +# @ingroup Translate # @BRIEF Get the latest preview session for a job with its records. def get_preview_session(db: Session, job_id: str) -> dict[str, Any]: """Get the latest preview session for a job with its records.""" diff --git a/backend/src/plugins/translate/preview_session_serializer.py b/backend/src/plugins/translate/preview_session_serializer.py index 5d30e50b..21f9804c 100644 --- a/backend/src/plugins/translate/preview_session_serializer.py +++ b/backend/src/plugins/translate/preview_session_serializer.py @@ -1,4 +1,5 @@ # #region preview_session_serializer [C:3] [TYPE Module] [SEMANTICS preview, record, serialize, action] +# @defgroup Translate Module group. # @BRIEF Serialization and action helpers for preview sessions: record serialization and per-row approve/reject/edit actions. # @RELATION DEPENDS_ON -> [TranslationPreviewRecord] # @RELATION DEPENDS_ON -> [TranslationPreviewLanguage] @@ -45,6 +46,7 @@ def serialize_preview_record(r: TranslationPreviewRecord) -> dict[str, Any]: # #region apply_language_action [C:2] [TYPE Function] [SEMANTICS preview, language, action] +# @ingroup Translate # @BRIEF Apply approve/reject/edit action to a TranslationPreviewLanguage entry. def apply_language_action( lang_entry: TranslationPreviewLanguage, @@ -68,6 +70,7 @@ def apply_language_action( # #region apply_record_action [C:2] [TYPE Function] [SEMANTICS preview, record, action] +# @ingroup Translate # @BRIEF Apply approve/reject/edit action to a TranslationPreviewRecord and its language entries. def apply_record_action( record: TranslationPreviewRecord, diff --git a/backend/src/plugins/translate/preview_token_estimator.py b/backend/src/plugins/translate/preview_token_estimator.py index 98d3706f..c67b765b 100644 --- a/backend/src/plugins/translate/preview_token_estimator.py +++ b/backend/src/plugins/translate/preview_token_estimator.py @@ -1,4 +1,5 @@ # #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS tokens, cost, estimation] +# @defgroup Translate Module group. # @BRIEF Estimate token counts and costs for LLM translation operations. # @RATIONALE Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model. # @REJECTED Using an external tokenizer library would introduce a heavy dependency for estimation only. diff --git a/backend/src/plugins/translate/prompt_builder.py b/backend/src/plugins/translate/prompt_builder.py index 174df28f..173b5e84 100644 --- a/backend/src/plugins/translate/prompt_builder.py +++ b/backend/src/plugins/translate/prompt_builder.py @@ -1,4 +1,5 @@ # #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary] +# @defgroup Translate Module group. # @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations. # @LAYER Domain # @RELATION DEPENDS_ON -> [DictionaryEntry] @@ -15,6 +16,7 @@ import json from typing import Any # #region ContextAwarePromptBuilder [C:2] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority. class ContextAwarePromptBuilder: diff --git a/backend/src/plugins/translate/scheduler.py b/backend/src/plugins/translate/scheduler.py index b529de77..63da6925 100644 --- a/backend/src/plugins/translate/scheduler.py +++ b/backend/src/plugins/translate/scheduler.py @@ -1,4 +1,5 @@ # #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, schedule, cron, job] +# @defgroup Translate Module group. # @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationSchedule] @@ -26,6 +27,7 @@ from .events import TranslationEventLog # #region TranslationScheduler [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers. class TranslationScheduler: @@ -241,6 +243,7 @@ class TranslationScheduler: # #region execute_scheduled_translation [C:4] [TYPE Function] +# @ingroup Translate # @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback. # @PRE schedule_id is valid. # @POST Translation run created and executed if no concurrent run exists. diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index d3178444..2774276d 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -1,4 +1,5 @@ # #region TranslateJobService [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, job, datasource, dialect] +# @defgroup Translate Module group. # @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationJob] @@ -26,6 +27,7 @@ from .service_utils import _extract_dialect # #region TranslateJobService [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Service for translation job CRUD with validation and Superset integration. class TranslateJobService: """CRUD service for translation jobs with Superset datasource integration.""" diff --git a/backend/src/plugins/translate/service_bulk_replace.py b/backend/src/plugins/translate/service_bulk_replace.py index adafbbd4..85b602fe 100644 --- a/backend/src/plugins/translate/service_bulk_replace.py +++ b/backend/src/plugins/translate/service_bulk_replace.py @@ -1,4 +1,5 @@ # #region BulkFindReplaceService [C:3] [TYPE Class] [SEMANTICS translate, bulk, find, replace, regex] +# @defgroup Translate Module group. # @BRIEF Service for bulk find-and-replace operations on translated values. # @RELATION DEPENDS_ON -> [TranslationLanguage] # @RELATION DEPENDS_ON -> [TranslationRecord] diff --git a/backend/src/plugins/translate/service_datasource.py b/backend/src/plugins/translate/service_datasource.py index 8ce7cd12..9607d128 100644 --- a/backend/src/plugins/translate/service_datasource.py +++ b/backend/src/plugins/translate/service_datasource.py @@ -1,4 +1,5 @@ # #region DatasourceMetadataService [C:4] [TYPE Module] [SEMANTICS superset, datasource, columns, dialect] +# @defgroup Translate Module group. # @BRIEF Fetch datasource column metadata and database dialect from Superset. # @LAYER Domain # @RELATION DEPENDS_ON -> [SupersetClient] @@ -22,6 +23,7 @@ SUPPORTED_DIALECTS = { # #region get_dialect_from_database [C:2] [TYPE Function] +# @ingroup Translate # @BRIEF Extract normalized dialect string from a Superset database record. def get_dialect_from_database(database_record: dict[str, Any]) -> str: """Extract and validate dialect from Superset database record.""" @@ -56,6 +58,7 @@ def get_dialect_from_database(database_record: dict[str, Any]) -> str: # #region fetch_datasource_metadata [C:3] [TYPE Function] +# @ingroup Translate # @BRIEF Fetch datasource columns and database dialect from Superset. async def fetch_datasource_metadata( dataset_id: int, @@ -105,6 +108,7 @@ async def fetch_datasource_metadata( # #region detect_virtual_columns [C:2] [TYPE Function] +# @ingroup Translate # @BRIEF Identify virtual (calculated) columns from column metadata. def detect_virtual_columns(columns: list[dict[str, Any]]) -> list[str]: """Return names of columns that are virtual (not physical).""" @@ -113,6 +117,7 @@ def detect_virtual_columns(columns: list[dict[str, Any]]) -> list[str]: # #region get_datasource_columns [C:3] [TYPE Function] +# @ingroup Translate # @BRIEF Fetch datasource column metadata from Superset and return structured response. async def get_datasource_columns( datasource_id: int, diff --git a/backend/src/plugins/translate/service_inline_correction.py b/backend/src/plugins/translate/service_inline_correction.py index f94038c9..3ce40e3e 100644 --- a/backend/src/plugins/translate/service_inline_correction.py +++ b/backend/src/plugins/translate/service_inline_correction.py @@ -1,4 +1,5 @@ # #region InlineCorrectionService [C:4] [TYPE Class] [SEMANTICS translate, correction, inline, dictionary] +# @defgroup Translate Module group. # @BRIEF Service for inline editing translated values and submitting corrections to dictionaries. # @PRE Database session is available. # @POST Inline edits applied with optional dictionary submission. diff --git a/backend/src/plugins/translate/service_target_schema.py b/backend/src/plugins/translate/service_target_schema.py index d1b57868..e6b8d67e 100644 --- a/backend/src/plugins/translate/service_target_schema.py +++ b/backend/src/plugins/translate/service_target_schema.py @@ -1,4 +1,5 @@ # #region TargetSchemaValidation [C:4] [TYPE Module] [SEMANTICS translate, schema, validation, target-table] +# @defgroup Translate Module group. # @BRIEF Проверка схемы целевой таблицы: запрос колонок через Superset SQL Lab, # сравнение с ожидаемыми (из build_columns), возврат diff. # @LAYER Service @@ -181,6 +182,7 @@ def _parse_sqllab_result(result: dict[str, Any]) -> tuple[list[dict[str, Any]], # #region validate_target_table_schema [C:4] [TYPE Function] [SEMANTICS translate, schema, validate, orchestrate] +# @ingroup Translate # @BRIEF Основная функция: проверяет схему целевой таблицы через Superset SQL Lab. # @PRE Superset окружение и target_database_id валидны. # @POST Возвращает TargetSchemaValidationResponse с diff-анализом. diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py index 95145678..be8d24a2 100644 --- a/backend/src/plugins/translate/sql_generator.py +++ b/backend/src/plugins/translate/sql_generator.py @@ -1,4 +1,5 @@ # #region SQLGenerator [C:3] [TYPE Module] [SEMANTICS clickhouse, translate, sql, insert, generate] +# @defgroup Translate Module group. # @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationJob] @@ -142,6 +143,7 @@ def _build_values_clause(columns: list[str], rows: list[dict[str, Any]], dialect # #region generate_insert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,insert] +# @ingroup Translate # @BRIEF Generate a dialect-aware plain INSERT SQL statement for the given table, columns, and rows. # @RELATION DEPENDS_ON -> [_quote_identifier] # @RELATION DEPENDS_ON -> [_build_values_clause] @@ -174,6 +176,7 @@ def generate_insert_sql( # #region generate_upsert_sql [C:4] [TYPE Function] +# @ingroup Translate # @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE. # @PRE dialect is postgresql-compatible. target_table, columns, key_columns are non-empty. # @POST Returns UPSERT SQL string or raises ValueError. @@ -229,6 +232,7 @@ def generate_upsert_sql( # #region SQLGenerator [C:3] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements. # @PRE Job has target_schema, target_table, key columns configured. # @POST Returns generated SQL string for the target dialect. diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index 28e0e1d7..1311102a 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -1,4 +1,5 @@ # #region SupersetSqlLabExecutor [C:4] [TYPE Module] [SEMANTICS translate, superset, sqllab, execute, query] +# @defgroup Translate Module group. # @BRIEF Submit SQL to Superset SQL Lab API and poll execution status. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [ConfigManager] @@ -21,6 +22,7 @@ from ...core.utils.client_registry import get_superset_client # #region SupersetSqlLabExecutor [C:4] [TYPE Class] +# @defgroup Translate Module group. # @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking. # @PRE Valid environment ID and ConfigManager. # @POST SQL submitted to Superset; execution reference recorded. diff --git a/backend/src/schemas/__init__.py b/backend/src/schemas/__init__.py index 1ccd5657..4f463112 100644 --- a/backend/src/schemas/__init__.py +++ b/backend/src/schemas/__init__.py @@ -1,3 +1,4 @@ # #region SchemasPackage [TYPE Package] [SEMANTICS schema, package, init] +# @ingroup Schemas # @BRIEF API schema package root. # #endregion SchemasPackage diff --git a/backend/src/schemas/auth.py b/backend/src/schemas/auth.py index db08ce0e..a6848bb2 100644 --- a/backend/src/schemas/auth.py +++ b/backend/src/schemas/auth.py @@ -1,4 +1,5 @@ # #region AuthSchemas [C:5] [TYPE Module] [SEMANTICS pydantic, auth, schema, token] +# @defgroup Schemas Module group. # # @BRIEF Pydantic schemas for authentication requests and responses. # @LAYER API @@ -47,6 +48,7 @@ class PermissionSchema(BaseModel): # #region RoleSchema [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Represents a role in API responses. class RoleSchema(BaseModel): id: str @@ -61,6 +63,7 @@ class RoleSchema(BaseModel): # #region RoleCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating a new role. class RoleCreate(BaseModel): name: str @@ -72,6 +75,7 @@ class RoleCreate(BaseModel): # #region RoleUpdate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for updating an existing role. class RoleUpdate(BaseModel): name: str | None = None @@ -83,6 +87,7 @@ class RoleUpdate(BaseModel): # #region ADGroupMappingSchema [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Represents an AD Group to Role mapping in API responses. class ADGroupMappingSchema(BaseModel): id: str @@ -96,6 +101,7 @@ class ADGroupMappingSchema(BaseModel): # #region ADGroupMappingCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating an AD Group mapping. class ADGroupMappingCreate(BaseModel): ad_group: str @@ -119,6 +125,7 @@ class ADGroupMappingCreate(BaseModel): # #region UserBase [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Base schema for user data. class UserBase(BaseModel): username: str @@ -130,6 +137,7 @@ class UserBase(BaseModel): # #region UserCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating a new user. class UserCreate(UserBase): password: str = Field(min_length=8) @@ -153,6 +161,7 @@ class UserCreate(UserBase): # #region UserUpdate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for updating an existing user. class UserUpdate(BaseModel): email: EmailStr | None = None @@ -165,6 +174,7 @@ class UserUpdate(BaseModel): # #region User [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for user data in API responses. class User(UserBase): id: str diff --git a/backend/src/schemas/dataset_review.py b/backend/src/schemas/dataset_review.py index d5db2179..7c1b6231 100644 --- a/backend/src/schemas/dataset_review.py +++ b/backend/src/schemas/dataset_review.py @@ -1,4 +1,5 @@ # #region DatasetReviewSchemas [C:2] [TYPE Module] [SEMANTICS dataset, review, schema, facade] +# @defgroup Schemas Module group. # @BRIEF Thin facade re-exporting all dataset review API schemas from decomposed sub-modules. # @LAYER API # @RATIONALE Original 419-line file exceeded INV_7 (400-line module limit). Decomposed into DTO and composite sub-modules. diff --git a/backend/src/schemas/dataset_review_pkg/_composites.py b/backend/src/schemas/dataset_review_pkg/_composites.py index 05c5b2a2..5b12017b 100644 --- a/backend/src/schemas/dataset_review_pkg/_composites.py +++ b/backend/src/schemas/dataset_review_pkg/_composites.py @@ -1,4 +1,5 @@ # #region DatasetReviewSchemaComposites [C:2] [TYPE Module] [SEMANTICS pydantic, dataset, review, composite, schema] +# @defgroup Schemas Module group. # @BRIEF Composite Pydantic DTOs for clarification, preview, run context, and session summary/detail responses. # @LAYER API # @RELATION DEPENDS_ON -> [DatasetReviewSchemaDtos] @@ -66,6 +67,7 @@ class ClarificationAnswerDto(BaseModel): # #region ClarificationQuestionDto [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Clarification question DTO with nested options and answer. class ClarificationQuestionDto(BaseModel): question_id: str @@ -88,6 +90,7 @@ class ClarificationQuestionDto(BaseModel): # #region ClarificationSessionDto [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Clarification session DTO with nested questions. class ClarificationSessionDto(BaseModel): clarification_session_id: str @@ -155,6 +158,7 @@ class DatasetRunContextDto(BaseModel): # #region SessionSummary [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Lightweight session summary DTO for list responses. class SessionSummary(BaseModel): session_id: str @@ -181,6 +185,7 @@ class SessionSummary(BaseModel): # #region SessionDetail [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Full session detail DTO with all nested aggregates for detail views. # @RELATION INHERITS -> [SessionSummary] class SessionDetail(SessionSummary): diff --git a/backend/src/schemas/dataset_review_pkg/_dtos.py b/backend/src/schemas/dataset_review_pkg/_dtos.py index 6e75012f..8e5ede08 100644 --- a/backend/src/schemas/dataset_review_pkg/_dtos.py +++ b/backend/src/schemas/dataset_review_pkg/_dtos.py @@ -1,4 +1,5 @@ # #region DatasetReviewSchemaDtos [C:2] [TYPE Module] [SEMANTICS pydantic, dataset, review, dto, schema] +# @defgroup Schemas Module group. # @BRIEF Pydantic DTOs for session, profile, findings, collaborators, and semantic field API payloads. # @LAYER API # @RELATION DEPENDS_ON -> [DatasetReviewModels] @@ -137,6 +138,7 @@ class SemanticCandidateDto(BaseModel): # #region SemanticFieldEntryDto [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Semantic field entry DTO with nested candidates and session version. class SemanticFieldEntryDto(BaseModel): field_id: str diff --git a/backend/src/schemas/health.py b/backend/src/schemas/health.py index bf8b161f..1f6922b4 100644 --- a/backend/src/schemas/health.py +++ b/backend/src/schemas/health.py @@ -1,4 +1,5 @@ # #region HealthSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, health, schema, dashboard, dashboard-health-item] +# @defgroup Schemas Module group. # @BRIEF Pydantic schemas for dashboard health summary — includes v2 LLM validation fields. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] @@ -10,6 +11,7 @@ from pydantic import BaseModel, Field # #region DashboardHealthItem [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Represents the latest health status of a single dashboard, with v2 LLM validation data. class DashboardHealthItem(BaseModel): record_id: str | None = None @@ -37,6 +39,7 @@ class DashboardHealthItem(BaseModel): # #region HealthSummaryResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Aggregated health summary for all dashboards. class HealthSummaryResponse(BaseModel): items: list[DashboardHealthItem] diff --git a/backend/src/schemas/profile.py b/backend/src/schemas/profile.py index e8fad9d3..92702e59 100644 --- a/backend/src/schemas/profile.py +++ b/backend/src/schemas/profile.py @@ -1,4 +1,5 @@ # #region ProfileSchemas [C:5] [TYPE Module] [SEMANTICS pydantic, profile, schema, superset, profile-permission-state] +# @defgroup Schemas Module group. # # @BRIEF Defines API schemas for profile preference persistence, security read-only snapshot, and Superset account lookup. # @LAYER API @@ -14,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field # #region ProfilePermissionState [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Represents one permission badge state for profile read-only security view. # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePermissionState(BaseModel): @@ -25,6 +27,7 @@ class ProfilePermissionState(BaseModel): # #region ProfileSecuritySummary [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Read-only security and access snapshot for current user. # @RELATION DEPENDS_ON -> ProfileSchemas class ProfileSecuritySummary(BaseModel): @@ -40,6 +43,7 @@ class ProfileSecuritySummary(BaseModel): # #region ProfilePreference [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Represents persisted profile preference for a single authenticated user. # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreference(BaseModel): @@ -72,6 +76,7 @@ class ProfilePreference(BaseModel): # #region ProfilePreferenceUpdateRequest [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Request payload for updating current user's profile settings. # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreferenceUpdateRequest(BaseModel): @@ -131,6 +136,7 @@ class ProfilePreferenceUpdateRequest(BaseModel): # #region ProfilePreferenceResponse [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Response envelope for profile preference read/update endpoints. # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreferenceResponse(BaseModel): @@ -145,6 +151,7 @@ class ProfilePreferenceResponse(BaseModel): # #region SupersetAccountLookupRequest [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Query contract for Superset account lookup by selected environment. # @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountLookupRequest(BaseModel): @@ -160,6 +167,7 @@ class SupersetAccountLookupRequest(BaseModel): # #region SupersetAccountCandidate [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Canonical account candidate projected from Superset users payload. # @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountCandidate(BaseModel): @@ -174,6 +182,7 @@ class SupersetAccountCandidate(BaseModel): # #region SupersetAccountLookupResponse [C:3] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Response envelope for Superset account lookup (success or degraded mode). # @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountLookupResponse(BaseModel): diff --git a/backend/src/schemas/settings.py b/backend/src/schemas/settings.py index 3fb4932c..0b5ea22e 100644 --- a/backend/src/schemas/settings.py +++ b/backend/src/schemas/settings.py @@ -1,4 +1,5 @@ # #region SettingsSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, schema, validate, notification-channel] +# @defgroup Schemas Module group. # @BRIEF Pydantic schemas for application settings and automation policies. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] @@ -9,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field # #region NotificationChannel [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Structured notification channel definition for policy-level custom routing. class NotificationChannel(BaseModel): type: str = Field( @@ -23,6 +25,7 @@ class NotificationChannel(BaseModel): # #region ValidationPolicyBase [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Base schema for validation policy data. class ValidationPolicyBase(BaseModel): name: str = Field(..., description="Name of the policy") @@ -53,6 +56,7 @@ class ValidationPolicyBase(BaseModel): # #region ValidationPolicyCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating a new validation policy. class ValidationPolicyCreate(ValidationPolicyBase): pass @@ -62,6 +66,7 @@ class ValidationPolicyCreate(ValidationPolicyBase): # #region ValidationPolicyUpdate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for updating an existing validation policy. class ValidationPolicyUpdate(BaseModel): name: str | None = None @@ -80,6 +85,7 @@ class ValidationPolicyUpdate(BaseModel): # #region ValidationPolicyResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for validation policy response data. class ValidationPolicyResponse(ValidationPolicyBase): id: str diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index 8efa2825..c443ada3 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -1,4 +1,5 @@ # #region TranslateSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, translate, schema, translate-job-create] +# @defgroup Schemas Module group. # @BRIEF Pydantic v2 schemas for translation API request/response serialization. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] @@ -26,6 +27,7 @@ def _validate_bcp47_list(v: list[str] | None) -> list[str] | None: # #region TranslateJobCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating a new translation job. class TranslateJobCreate(BaseModel): name: str @@ -60,6 +62,7 @@ class TranslateJobCreate(BaseModel): # #region TranslateJobUpdate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for updating an existing translation job. class TranslateJobUpdate(BaseModel): name: str | None = None @@ -95,6 +98,7 @@ class TranslateJobUpdate(BaseModel): # #region TranslateJobResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for translation job API responses. class TranslateJobResponse(BaseModel): id: str @@ -134,6 +138,7 @@ class TranslateJobResponse(BaseModel): # #region DatasourceColumnResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for datasource column metadata response. class DatasourceColumnResponse(BaseModel): name: str @@ -145,6 +150,7 @@ class DatasourceColumnResponse(BaseModel): # #endregion DatasourceColumnResponse # #region DatasourceColumnsResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for datasource columns endpoint response. class DatasourceColumnsResponse(BaseModel): datasource_id: int @@ -158,6 +164,7 @@ class DatasourceColumnsResponse(BaseModel): # #region DuplicateJobResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for duplicate job response. class DuplicateJobResponse(BaseModel): id: str @@ -169,6 +176,7 @@ class DuplicateJobResponse(BaseModel): # #region DictionaryCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating a new terminology dictionary. class DictionaryCreate(BaseModel): name: str @@ -178,6 +186,7 @@ class DictionaryCreate(BaseModel): # #region DictionaryImport [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for importing entries into a terminology dictionary. class DictionaryImport(BaseModel): content: str = Field(..., description="CSV or TSV content as raw string") @@ -190,6 +199,7 @@ class DictionaryImport(BaseModel): # #region DictionaryResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for terminology dictionary API responses. class DictionaryResponse(BaseModel): id: str @@ -206,6 +216,7 @@ class DictionaryResponse(BaseModel): # #region DictionaryEntryCreate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for adding/editing a dictionary entry with language pair support. class DictionaryEntryCreate(BaseModel): source_term: str = Field(..., description="Source term to translate") @@ -220,6 +231,7 @@ class DictionaryEntryCreate(BaseModel): # #region DictionaryEntryResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for dictionary entry API responses. class DictionaryEntryResponse(BaseModel): id: str @@ -244,6 +256,7 @@ class DictionaryEntryResponse(BaseModel): # #region DictionaryImportResult [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for dictionary import result. class DictionaryImportResult(BaseModel): total: int = 0 @@ -256,6 +269,7 @@ class DictionaryImportResult(BaseModel): # #region PreviewRequest [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for triggering a translation preview. class PreviewRequest(BaseModel): sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview") @@ -265,6 +279,7 @@ class PreviewRequest(BaseModel): # #region PreviewRowUpdate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for approving/editing/rejecting a preview row. class PreviewRowUpdate(BaseModel): action: str = Field(..., description="'approve', 'reject', or 'edit'") @@ -275,6 +290,7 @@ class PreviewRowUpdate(BaseModel): # #region PreviewAcceptResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for preview accept response. class PreviewAcceptResponse(BaseModel): id: str @@ -291,6 +307,7 @@ class PreviewAcceptResponse(BaseModel): # #region CostEstimate [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for cost estimation in preview response. class CostEstimate(BaseModel): sample_size: int = 0 @@ -307,6 +324,7 @@ class CostEstimate(BaseModel): # #region TermCorrectionSubmit [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for submitting a term correction in a translation preview. class TermCorrectionSubmit(BaseModel): source_term: str @@ -322,6 +340,7 @@ class TermCorrectionSubmit(BaseModel): # #region TermCorrectionBulkSubmit [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for submitting multiple term corrections atomically. class TermCorrectionBulkSubmit(BaseModel): corrections: list[TermCorrectionSubmit] @@ -330,6 +349,7 @@ class TermCorrectionBulkSubmit(BaseModel): # #region CorrectionConflictResult [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for reporting conflicts in correction submission. class CorrectionConflictResult(BaseModel): source_term: str @@ -340,6 +360,7 @@ class CorrectionConflictResult(BaseModel): # #region CorrectionSubmitResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for correction submission response. class CorrectionSubmitResponse(BaseModel): entry_id: str | None = None @@ -352,6 +373,7 @@ class CorrectionSubmitResponse(BaseModel): # #region ScheduleConfig [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for configuring a recurring translation schedule. class ScheduleConfig(BaseModel): cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')") @@ -362,6 +384,7 @@ class ScheduleConfig(BaseModel): # #region ScheduleResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for schedule API responses. class ScheduleResponse(BaseModel): id: str @@ -381,6 +404,7 @@ class ScheduleResponse(BaseModel): # #region NextExecutionResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for next execution preview. class NextExecutionResponse(BaseModel): job_id: str @@ -391,6 +415,7 @@ class NextExecutionResponse(BaseModel): # #region TranslationRunResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for translation run API responses. class TranslationRunResponse(BaseModel): id: str @@ -420,6 +445,7 @@ class TranslationRunResponse(BaseModel): # #region RunDetailResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for detailed run response including records, events, and config snapshot. class RunDetailResponse(BaseModel): run: TranslationRunResponse @@ -431,6 +457,7 @@ class RunDetailResponse(BaseModel): # #region RunHistoryFilter [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for filtering run history. class RunHistoryFilter(BaseModel): job_id: str | None = None @@ -445,6 +472,7 @@ class RunHistoryFilter(BaseModel): # #region RunListResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for paginated run list response. class RunListResponse(BaseModel): items: list[TranslationRunResponse] = [] @@ -455,6 +483,7 @@ class RunListResponse(BaseModel): # #region AggregatedMetricsResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for aggregated per-job metrics. class AggregatedMetricsResponse(BaseModel): job_id: str @@ -490,6 +519,7 @@ class TranslationPreviewLanguageResponse(BaseModel): # #region PreviewRow [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF A single row in a translation preview showing original and translated content side by side. class PreviewRow(BaseModel): id: str @@ -507,6 +537,7 @@ class PreviewRow(BaseModel): # #region TranslationPreviewResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for translation preview session API responses. class TranslationPreviewResponse(BaseModel): id: str @@ -525,6 +556,7 @@ class TranslationPreviewResponse(BaseModel): # #region TranslationBatchResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for translation batch API responses. class TranslationBatchResponse(BaseModel): id: str @@ -543,6 +575,7 @@ class TranslationBatchResponse(BaseModel): # #region MetricsResponse [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for translation metrics API responses. class MetricsResponse(BaseModel): job_id: str diff --git a/backend/src/schemas/validation.py b/backend/src/schemas/validation.py index 641be1bc..5dec0aae 100644 --- a/backend/src/schemas/validation.py +++ b/backend/src/schemas/validation.py @@ -1,4 +1,5 @@ # #region ValidationSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, validation, schema, task, run] +# @defgroup Schemas Module group. # @BRIEF Pydantic v2 schemas for validation task management and run history API serialization. # @LAYER API # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] @@ -41,6 +42,7 @@ class SourceResponse(BaseModel): # #region ValidationTaskCreate [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for creating a new validation task (policy). v2 adds sources, screenshot_enabled, # logs_enabled, execute_chart_data, llm_batch_size, concurrency_limit, and prompt_template. # @RATIONALE All v2 fields are optional to support gradual migration. dashboard_ids is kept for @@ -73,6 +75,7 @@ class ValidationTaskCreate(BaseModel): # #region ValidationTaskUpdate [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for updating an existing validation task (policy). Includes v2 fields. # @RATIONALE All fields optional (partial update). When sources is provided, replaces all existing # ValidationSource rows for the policy. @@ -102,6 +105,7 @@ class ValidationTaskUpdate(BaseModel): # #region ValidationTaskResponse [C:2] [TYPE Class] +# @defgroup Schemas Module group. # @BRIEF Schema for validation task API responses — includes last run info and v2 fields. class ValidationTaskResponse(BaseModel): id: str diff --git a/backend/src/scripts/__init__.py b/backend/src/scripts/__init__.py index ee7232a3..cae1b36c 100644 --- a/backend/src/scripts/__init__.py +++ b/backend/src/scripts/__init__.py @@ -1,3 +1,4 @@ # #region ScriptsPackage [TYPE Package] +# @ingroup Module # @BRIEF Script entrypoint package root. # #endregion ScriptsPackage diff --git a/backend/src/scripts/check_migration_chain.py b/backend/src/scripts/check_migration_chain.py index e931e91b..6240f435 100644 --- a/backend/src/scripts/check_migration_chain.py +++ b/backend/src/scripts/check_migration_chain.py @@ -1,4 +1,5 @@ # #region check_migration_chain [C:3] [TYPE Module] [SEMANTICS alembic,migration,validation,ci] +# @defgroup Module Module group. # @BRIEF Validates Alembic migration chain integrity — checks for multiple heads, # broken chains, and missing down_revision references. # @RATIONALE Merge migrations (e.g., f0e9d8c7b6a5) are fragile — a new migration @@ -15,6 +16,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # #region check_migration_heads [C:2] [TYPE Function] +# @ingroup Module # @BRIEF Checks that there is exactly one migration head. # @POST Prints error and exits 1 if multiple heads found. def check_migration_heads() -> None: @@ -45,6 +47,7 @@ def check_migration_heads() -> None: # #endregion check_migration_heads # #region check_migration_chain_integrity [C:2] [TYPE Function] +# @ingroup Module # @BRIEF Walks the migration chain from head to root, checking all down_revision links exist. # @POST Prints error and exits 1 if a broken link is found. def check_migration_chain_integrity() -> None: @@ -100,6 +103,7 @@ def check_migration_chain_integrity() -> None: # #endregion check_migration_chain_integrity # #region check_migration_main [C:2] [TYPE Function] +# @ingroup Module # @BRIEF Entry point — runs all migration chain checks. def main() -> None: print("[migration-check] === Alembic Migration Chain Validation ===") diff --git a/backend/src/scripts/clean_release_cli.py b/backend/src/scripts/clean_release_cli.py index bbe855ab..17ae8bec 100644 --- a/backend/src/scripts/clean_release_cli.py +++ b/backend/src/scripts/clean_release_cli.py @@ -1,4 +1,5 @@ # #region CleanReleaseCliScript [C:3] [TYPE Module] [SEMANTICS clean-release, artifact, manifest, candidate, cli] +# @defgroup Module Module group. # @BRIEF Provide headless CLI commands for candidate registration, artifact import and manifest build. # @LAYER Service # @RELATION CALLS -> ComplianceOrchestrator @@ -27,6 +28,7 @@ from ..services.clean_release.publication_service import ( # #region build_parser [TYPE Function] +# @ingroup Module # @BRIEF Build argparse parser for clean release CLI. def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="clean-release-cli") @@ -102,6 +104,7 @@ def build_parser() -> argparse.ArgumentParser: # #region run_candidate_register [TYPE Function] +# @ingroup Module # @BRIEF Register candidate in repository via CLI command. # @PRE Candidate ID must be unique. # @POST Candidate is persisted in DRAFT status. @@ -131,6 +134,7 @@ def run_candidate_register(args: argparse.Namespace) -> int: # #region run_artifact_import [TYPE Function] +# @ingroup Module # @BRIEF Import single artifact for existing candidate. # @PRE Candidate must exist. # @POST Artifact is persisted for candidate. @@ -164,6 +168,7 @@ def run_artifact_import(args: argparse.Namespace) -> int: # #region run_manifest_build [TYPE Function] +# @ingroup Module # @BRIEF Build immutable manifest snapshot for candidate. # @PRE Candidate must exist. # @POST New manifest version is persisted. @@ -198,6 +203,7 @@ def run_manifest_build(args: argparse.Namespace) -> int: # #region run_compliance_run [TYPE Function] +# @ingroup Module # @BRIEF Execute compliance run for candidate with optional manifest fallback. # @PRE Candidate exists and trusted snapshots are configured. # @POST Returns run payload and exit code 0 on success. @@ -237,6 +243,7 @@ def run_compliance_run(args: argparse.Namespace) -> int: # #region run_compliance_status [TYPE Function] +# @ingroup Module # @BRIEF Read run status by run id. # @PRE Run exists. # @POST Returns run status payload. @@ -299,6 +306,7 @@ def _to_payload(value: Any) -> dict[str, Any]: # #region run_compliance_report [TYPE Function] +# @ingroup Module # @BRIEF Read immutable report by run id. # @PRE Run and report exist. # @POST Returns report payload. @@ -326,6 +334,7 @@ def run_compliance_report(args: argparse.Namespace) -> int: # #region run_compliance_violations [TYPE Function] +# @ingroup Module # @BRIEF Read run violations by run id. # @PRE Run exists. # @POST Returns violations payload. @@ -351,6 +360,7 @@ def run_compliance_violations(args: argparse.Namespace) -> int: # #region run_approve [TYPE Function] +# @ingroup Module # @BRIEF Approve candidate based on immutable PASSED report. # @PRE Candidate and report exist; report is PASSED. # @POST Persists APPROVED decision and returns success payload. @@ -382,6 +392,7 @@ def run_approve(args: argparse.Namespace) -> int: # #region run_reject [TYPE Function] +# @ingroup Module # @BRIEF Reject candidate without mutating compliance evidence. # @PRE Candidate and report exist. # @POST Persists REJECTED decision and returns success payload. @@ -413,6 +424,7 @@ def run_reject(args: argparse.Namespace) -> int: # #region run_publish [TYPE Function] +# @ingroup Module # @BRIEF Publish approved candidate to target channel. # @PRE Candidate is approved and report belongs to candidate. # @POST Appends ACTIVE publication record and returns payload. @@ -441,6 +453,7 @@ def run_publish(args: argparse.Namespace) -> int: # #region run_revoke [TYPE Function] +# @ingroup Module # @BRIEF Revoke active publication record. # @PRE Publication id exists and is ACTIVE. # @POST Publication record status becomes REVOKED. @@ -467,6 +480,7 @@ def run_revoke(args: argparse.Namespace) -> int: # #region main [TYPE Function] +# @ingroup Module # @BRIEF CLI entrypoint for clean release commands. def main(argv: list[str] | None = None) -> int: seed_trace_id() diff --git a/backend/src/scripts/clean_release_tui.py b/backend/src/scripts/clean_release_tui.py index 6b8c15c7..75b1cec0 100644 --- a/backend/src/scripts/clean_release_tui.py +++ b/backend/src/scripts/clean_release_tui.py @@ -1,4 +1,5 @@ # #region CleanReleaseTuiScript [C:3] [TYPE Module] [SEMANTICS clean-release, validate, compliance, release, tui-facade-adapter] +# @defgroup Module Module group. # @BRIEF Interactive terminal interface for Enterprise Clean Release compliance validation. # @LAYER UI # @RELATION DEPENDS_ON -> [ComplianceExecutionService] @@ -49,6 +50,7 @@ from src.services.clean_release.repository import CleanReleaseRepository # #region TuiFacadeAdapter [TYPE Class] +# @defgroup Module Module group. # @BRIEF Thin TUI adapter that routes business mutations through application services. # @PRE repository contains candidate and trusted policy/registry snapshots for execution. # @POST Business actions return service results/errors without direct TUI-owned mutations. @@ -185,6 +187,7 @@ class TuiFacadeAdapter: } # #endregion TuiFacadeAdapter # #region CleanReleaseTUI [TYPE Class] +# @defgroup Module Module group. # @BRIEF Curses-based application for compliance monitoring. # @UX_STATE READY -> Waiting for operator to start checks (F5). # @UX_STATE RUNNING -> Executing compliance stages with progress feedback. @@ -562,6 +565,7 @@ class CleanReleaseTUI: self.refresh_overview() self.refresh_screen() # #region bundle_build_mode [C:4] [TYPE Function] [SEMANTICS bundle,build,release] +# @ingroup Module # @BRIEF Interactive bundle build screen — select type, enter tag, watch live build output. # @PRE TTY is available. Bundle type selected (1 or 2). Tag is non-empty. # @POST Subprocess completes (success/failure). Output displayed. User returns via Esc. diff --git a/backend/src/scripts/create_admin.py b/backend/src/scripts/create_admin.py index 0f2d2325..49b2aec2 100644 --- a/backend/src/scripts/create_admin.py +++ b/backend/src/scripts/create_admin.py @@ -1,4 +1,5 @@ # #region CreateAdminScript [C:5] [TYPE Module] [SEMANTICS admin, search, user, cli] +# @defgroup Module Module group. # # @BRIEF CLI tool for creating the initial admin user. # @LAYER Infrastructure @@ -25,6 +26,7 @@ from src.models.auth import Role, User # #region create_admin [TYPE Function] +# @ingroup Module # @BRIEF Creates an admin user and necessary roles/permissions. # @PRE username and password provided via CLI. # @POST Admin user exists in auth.db. diff --git a/backend/src/scripts/delete_running_tasks.py b/backend/src/scripts/delete_running_tasks.py index 413b08b5..a8c5a353 100644 --- a/backend/src/scripts/delete_running_tasks.py +++ b/backend/src/scripts/delete_running_tasks.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # #region DeleteRunningTasksUtil [C:3] [TYPE Module] +# @defgroup Module Module group. # @PURPOSE Script to delete tasks with RUNNING status from the database. # @LAYER Infrastructure # @SEMANTICS maintenance, database, cleanup @@ -11,6 +12,7 @@ from src.core.database import TasksSessionLocal from src.models.task import TaskRecord # #region delete_running_tasks [C:4] [TYPE Function] +# @ingroup Module # @PURPOSE Delete all tasks with RUNNING status from the database. # @PRE Database is accessible and TaskRecord model is defined. # @POST All tasks with status 'RUNNING' are removed from the database. diff --git a/backend/src/scripts/init_auth_db.py b/backend/src/scripts/init_auth_db.py index feaa1b43..041a9f11 100644 --- a/backend/src/scripts/init_auth_db.py +++ b/backend/src/scripts/init_auth_db.py @@ -1,4 +1,5 @@ # #region InitAuthDbScript [C:2] [TYPE Module] [SEMANTICS auth] +# @defgroup Module Module group. # # @BRIEF Initializes the auth database and creates the necessary tables. # @LAYER Service @@ -22,6 +23,7 @@ from src.scripts.seed_permissions import seed_permissions # #region run_init [C:3] [TYPE Function] +# @ingroup Module # @BRIEF Main entry point for the initialization script. # @POST auth.db is initialized with the correct schema and seeded permissions. # @RELATION CALLS -> [ensure_encryption_key] diff --git a/backend/src/scripts/seed_permissions.py b/backend/src/scripts/seed_permissions.py index f3efd7af..d22c4b47 100644 --- a/backend/src/scripts/seed_permissions.py +++ b/backend/src/scripts/seed_permissions.py @@ -1,4 +1,5 @@ # #region SeedPermissionsScript [C:5] [TYPE Module] [SEMANTICS rbac, search, auth] +# @defgroup Module Module group. # # @BRIEF Populates the auth database with initial system permissions. # @LAYER Infrastructure @@ -24,6 +25,7 @@ from src.core.logger import belief_scope, logger from src.models.auth import Permission, Role # #region INITIAL_PERMISSIONS [C:3] [TYPE Constant] +# @ingroup Module # @BRIEF Canonical bootstrap permission tuples seeded into auth storage. # @RELATION DEPENDS_ON -> SeedPermissionsScript INITIAL_PERMISSIONS = [ @@ -61,6 +63,7 @@ INITIAL_PERMISSIONS = [ # #region seed_permissions [C:3] [TYPE Function] +# @ingroup Module # @BRIEF Inserts missing permissions into the database. # @POST All INITIAL_PERMISSIONS exist in the DB. # @RELATION DEPENDS_ON -> AuthSessionLocal diff --git a/backend/src/scripts/seed_superset_load_test.py b/backend/src/scripts/seed_superset_load_test.py index a0135cb0..d73e1dba 100644 --- a/backend/src/scripts/seed_superset_load_test.py +++ b/backend/src/scripts/seed_superset_load_test.py @@ -1,4 +1,5 @@ # #region SeedSupersetLoadTestScript [C:5] [TYPE Module] [SEMANTICS superset, validate] +# @defgroup Module Module group. # # @BRIEF Creates randomized load-test data in Superset by cloning chart configurations and creating dashboards in target environments. # @LAYER Infrastructure @@ -241,6 +242,7 @@ def _build_chart_template_pool( # #region seed_superset_load_data [TYPE Function] +# @ingroup Module # @BRIEF Creates dashboards and cloned charts for load testing across target environments. # @PRE Target environments must be reachable and authenticated. # @POST Returns execution statistics dictionary. @@ -369,6 +371,7 @@ def seed_superset_load_data(args: argparse.Namespace) -> dict: # #region main [TYPE Function] +# @ingroup Module # @BRIEF CLI entrypoint for Superset load-test data seeding. # @PRE Command line arguments are valid. # @POST Prints summary and exits with non-zero status on failure. diff --git a/backend/src/scripts/test_dataset_dashboard_relations.py b/backend/src/scripts/test_dataset_dashboard_relations.py index 83b76f75..eb5bb6b1 100644 --- a/backend/src/scripts/test_dataset_dashboard_relations.py +++ b/backend/src/scripts/test_dataset_dashboard_relations.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # #region test_dataset_dashboard_relations_script [C:2] [TYPE Module] [SEMANTICS pydantic, dashboard, dataset, test, superset] +# @defgroup Module Module group. # @BRIEF Tests and inspects dataset-to-dashboard relationship responses from Superset API. # @RATIONALE Added '# DIAGNOSTIC SCRIPT — not a test' header comment. Script already had if __name__ guard. diff --git a/backend/src/services/__init__.py b/backend/src/services/__init__.py index 7977f75d..401880ca 100644 --- a/backend/src/services/__init__.py +++ b/backend/src/services/__init__.py @@ -1,4 +1,5 @@ # #region services [C:2] [TYPE Module] [SEMANTICS lazy, export, package, circular-import] +# @defgroup Services Module group. # @BRIEF Package initialization for services module # @NOTE: Only export services that don't cause circular imports # @NOTE: GitService, AuthService, LLMProviderService have circular import issues - import directly when needed diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index 12abef1f..8121be73 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -1,4 +1,5 @@ # #region auth_service [C:5] [TYPE Module] [SEMANTICS sqlalchemy, auth, credential, session, jwt] +# @defgroup Services Module group. # @BRIEF Orchestrates credential authentication and ADFS JIT user provisioning. # @LAYER Domain # @RELATION DEPENDS_ON -> [AuthRepository] @@ -26,6 +27,7 @@ from ..models.auth import User # #region AuthService [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Provides high-level authentication services. # @RELATION DEPENDS_ON -> [AuthRepository] # @RELATION DEPENDS_ON -> [User] diff --git a/backend/src/services/clean_release/__init__.py b/backend/src/services/clean_release/__init__.py index 450c7c50..0fe85335 100644 --- a/backend/src/services/clean_release/__init__.py +++ b/backend/src/services/clean_release/__init__.py @@ -1,4 +1,5 @@ # #region CleanReleaseContracts [C:3] [TYPE Module] [SEMANTICS clean-release, compliance, package, subsystem] +# @defgroup Services Module group. # @BRIEF Publish the canonical semantic root for the clean-release backend service cluster. # @LAYER Domain # @RELATION DEPENDS_ON -> [ComplianceOrchestrator] diff --git a/backend/src/services/clean_release/approval_service.py b/backend/src/services/clean_release/approval_service.py index 3046f2cc..3630ff0f 100644 --- a/backend/src/services/clean_release/approval_service.py +++ b/backend/src/services/clean_release/approval_service.py @@ -1,4 +1,5 @@ # #region ApprovalService [C:5] [TYPE Module] [SEMANTICS clean-release, approval, gate, compliance] +# @defgroup Services Module group. # @BRIEF Enforce approval/rejection gates over immutable compliance reports. # @LAYER Domain # @RELATION DEPENDS_ON -> [RepositoryRelations] @@ -89,6 +90,7 @@ def _resolve_candidate_and_report( # #region approve_candidate [TYPE Function] +# @ingroup Services # @BRIEF Persist immutable APPROVED decision and advance candidate lifecycle to APPROVED. # @PRE Candidate exists, report belongs to candidate, report final_status is PASSED, candidate not already APPROVED. # @POST Approval decision is appended and candidate transitions to APPROVED. @@ -165,6 +167,7 @@ def approve_candidate( # #region reject_candidate [TYPE Function] +# @ingroup Services # @BRIEF Persist immutable REJECTED decision without promoting candidate lifecycle. # @PRE Candidate exists and report belongs to candidate. # @POST Rejected decision is appended; candidate lifecycle is unchanged. diff --git a/backend/src/services/clean_release/artifact_catalog_loader.py b/backend/src/services/clean_release/artifact_catalog_loader.py index ffea5f07..769f9868 100644 --- a/backend/src/services/clean_release/artifact_catalog_loader.py +++ b/backend/src/services/clean_release/artifact_catalog_loader.py @@ -1,4 +1,5 @@ # #region ArtifactCatalogLoader [C:5] [TYPE Module] [SEMANTICS pydantic, clean-release, artifact, catalog, manifest] +# @defgroup Services Module group. # @BRIEF Load bootstrap artifact catalogs for clean release real-mode flows. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] @@ -15,6 +16,7 @@ from ...models.clean_release import CandidateArtifact # #region load_bootstrap_artifacts [TYPE Function] +# @ingroup Services # @BRIEF Parse artifact catalog JSON into CandidateArtifact models for TUI/bootstrap flows. # @PRE path points to readable JSON file; payload is list[artifact] or {"artifacts": list[artifact]}. # @POST Returns non-mutated CandidateArtifact models with required fields populated. diff --git a/backend/src/services/clean_release/audit_service.py b/backend/src/services/clean_release/audit_service.py index 58802077..6dec31a2 100644 --- a/backend/src/services/clean_release/audit_service.py +++ b/backend/src/services/clean_release/audit_service.py @@ -1,4 +1,5 @@ # #region AuditService [C:3] [TYPE Module] [SEMANTICS clean-release, audit, report, trail, release] +# @defgroup Services Module group. # @BRIEF Provide lightweight audit hooks for clean release preparation/check/report lifecycle. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [LoggerModule] diff --git a/backend/src/services/clean_release/candidate_service.py b/backend/src/services/clean_release/candidate_service.py index 79fa281f..22f22899 100644 --- a/backend/src/services/clean_release/candidate_service.py +++ b/backend/src/services/clean_release/candidate_service.py @@ -1,4 +1,5 @@ # #region candidate_service [C:5] [TYPE Module] [SEMANTICS pydantic, clean-release, candidate, lifecycle, release] +# @defgroup Services Module group. # @BRIEF Register release candidates with validated artifacts and advance lifecycle through legal transitions. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseRepository] @@ -47,6 +48,7 @@ def _validate_artifacts(artifacts: Iterable[dict[str, Any]]) -> list[dict[str, A # #region register_candidate [TYPE Function] +# @ingroup Services # @BRIEF Register a candidate and persist its artifacts with legal lifecycle transition. # @PRE candidate_id must be unique and artifacts must pass validation. # @POST Candidate exists in repository with PREPARED status and artifacts persisted. diff --git a/backend/src/services/clean_release/compliance_execution_service.py b/backend/src/services/clean_release/compliance_execution_service.py index 7efec972..92961e88 100644 --- a/backend/src/services/clean_release/compliance_execution_service.py +++ b/backend/src/services/clean_release/compliance_execution_service.py @@ -1,4 +1,5 @@ # #region ComplianceExecutionService [C:5] [TYPE Module] [SEMANTICS clean-release, execution, compliance, report, stage] +# @defgroup Services Module group. # @BRIEF Create and execute compliance runs with trusted snapshots, deterministic stages, violations and immutable report persistence. # @LAYER Domain # @RELATION DEPENDS_ON -> RepositoryRelations @@ -39,6 +40,7 @@ belief_logger = cast(Any, logger) # #region ComplianceExecutionResult [TYPE Class] +# @defgroup Services Module group. # @BRIEF Return envelope for compliance execution with run/report and persisted stage artifacts. @dataclass class ComplianceExecutionResult: @@ -52,6 +54,7 @@ class ComplianceExecutionResult: # #region ComplianceExecutionService [TYPE Class] +# @defgroup Services Module group. # @BRIEF Execute clean-release compliance lifecycle over trusted snapshots and immutable evidence. # @PRE Database session active, candidate registered # @POST Returns ComplianceReport with pass/fail status and violation details diff --git a/backend/src/services/clean_release/compliance_orchestrator.py b/backend/src/services/clean_release/compliance_orchestrator.py index f7668a22..d862c7df 100644 --- a/backend/src/services/clean_release/compliance_orchestrator.py +++ b/backend/src/services/clean_release/compliance_orchestrator.py @@ -1,4 +1,5 @@ # #region ComplianceOrchestrator [C:5] [TYPE Module] [SEMANTICS clean-release, compliance, orchestration, stage] +# @defgroup Services Module group. # @BRIEF Execute mandatory clean compliance stages and produce final COMPLIANT/BLOCKED/FAILED outcome. # @LAYER Domain # @RELATION DEPENDS_ON -> [ComplianceStages] @@ -35,6 +36,7 @@ from .stages import derive_final_status # #region CleanComplianceOrchestrator [TYPE Class] +# @defgroup Services Module group. # @BRIEF Coordinate clean-release compliance verification stages. class CleanComplianceOrchestrator: # region __init__ [TYPE Function] @@ -238,6 +240,7 @@ class CleanComplianceOrchestrator: # #region run_check_legacy [TYPE Function] +# @ingroup Services # @BRIEF Legacy wrapper for compatibility with previous orchestrator call style. # @PRE repository and identifiers are valid and resolvable by orchestrator dependencies. # @POST Returns finalized ComplianceRun produced by orchestrator start->execute->finalize sequence. diff --git a/backend/src/services/clean_release/demo_data_service.py b/backend/src/services/clean_release/demo_data_service.py index 9c12aa85..b7d579ff 100644 --- a/backend/src/services/clean_release/demo_data_service.py +++ b/backend/src/services/clean_release/demo_data_service.py @@ -1,4 +1,5 @@ # #region DemoDataService [C:5] [TYPE Module] [SEMANTICS clean-release, demo, seed, fixture] +# @defgroup Services Module group. # @BRIEF Provide deterministic namespace helpers and isolated in-memory repository creation for demo and real modes. # @LAYER Domain # @RELATION DEPENDS_ON -> [RepositoryRelations] @@ -12,6 +13,7 @@ from .repository import CleanReleaseRepository # #region resolve_namespace [TYPE Function] +# @ingroup Services # @BRIEF Resolve canonical clean-release namespace for requested mode. # @PRE mode is a non-empty string identifying runtime mode. # @POST Returns deterministic namespace key for demo/real separation. @@ -26,6 +28,7 @@ def resolve_namespace(mode: str) -> str: # #region build_namespaced_id [TYPE Function] +# @ingroup Services # @BRIEF Build storage-safe physical identifier under mode namespace. # @PRE namespace and logical_id are non-empty strings. # @POST Returns deterministic "{namespace}::{logical_id}" identifier. @@ -41,6 +44,7 @@ def build_namespaced_id(namespace: str, logical_id: str) -> str: # #region create_isolated_repository [TYPE Function] +# @ingroup Services # @BRIEF Create isolated in-memory repository instance for selected mode namespace. # @PRE mode is a valid runtime mode marker. # @POST Returns repository instance tagged with namespace metadata. diff --git a/backend/src/services/clean_release/dto.py b/backend/src/services/clean_release/dto.py index 42c897b2..21f997df 100644 --- a/backend/src/services/clean_release/dto.py +++ b/backend/src/services/clean_release/dto.py @@ -1,4 +1,5 @@ # #region clean_release_dto [C:3] [TYPE Module] [SEMANTICS pydantic, clean-release, dto, schema, transfer] +# @defgroup Services Module group. # @BRIEF Data Transfer Objects for clean release compliance subsystem. # @LAYER Application # @RELATION DEPENDS_ON -> [EXT:Library:pydantic] diff --git a/backend/src/services/clean_release/enums.py b/backend/src/services/clean_release/enums.py index 71d47030..2e6430d0 100644 --- a/backend/src/services/clean_release/enums.py +++ b/backend/src/services/clean_release/enums.py @@ -1,4 +1,5 @@ # #region clean_release_enums [C:3] [TYPE Module] [SEMANTICS clean-release, enum, lifecycle, status, compliance] +# @defgroup Services Module group. # @BRIEF Canonical enums for clean release lifecycle and compliance. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:Python:enum] diff --git a/backend/src/services/clean_release/exceptions.py b/backend/src/services/clean_release/exceptions.py index 1d2ad445..5f5144f7 100644 --- a/backend/src/services/clean_release/exceptions.py +++ b/backend/src/services/clean_release/exceptions.py @@ -1,4 +1,5 @@ # #region clean_release_exceptions [C:3] [TYPE Module] [SEMANTICS clean-release, exception, domain, error] +# @defgroup Services Module group. # @BRIEF Domain exceptions for clean release compliance subsystem. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:Python:Exception] diff --git a/backend/src/services/clean_release/facade.py b/backend/src/services/clean_release/facade.py index 0bc33148..7d80328a 100644 --- a/backend/src/services/clean_release/facade.py +++ b/backend/src/services/clean_release/facade.py @@ -1,4 +1,5 @@ # #region clean_release_facade [C:3] [TYPE Module] [SEMANTICS clean-release, facade, orchestration, repository, crud] +# @defgroup Services Module group. # @BRIEF Unified entry point for clean release operations. # @LAYER Application # @RELATION DEPENDS_ON -> ComplianceOrchestrator diff --git a/backend/src/services/clean_release/manifest_builder.py b/backend/src/services/clean_release/manifest_builder.py index cbadcdcc..519a4fde 100644 --- a/backend/src/services/clean_release/manifest_builder.py +++ b/backend/src/services/clean_release/manifest_builder.py @@ -1,4 +1,5 @@ # #region ManifestBuilder [C:5] [TYPE Module] [SEMANTICS clean-release, manifest, build, artifact, catalog] +# @defgroup Services Module group. # @BRIEF Build deterministic distribution manifest from classified artifact input. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] @@ -56,6 +57,7 @@ def _stable_hash_payload( # #region build_distribution_manifest [TYPE Function] +# @ingroup Services # @BRIEF Build DistributionManifest with deterministic hash and validated counters. # @PRE artifacts list contains normalized classification values. # @POST Returns DistributionManifest with summary counts matching items cardinality. @@ -113,6 +115,7 @@ def build_distribution_manifest( # #region build_manifest [TYPE Function] +# @ingroup Services # @BRIEF Legacy compatibility wrapper for old manifest builder import paths. # @PRE Same as build_distribution_manifest. # @POST Returns DistributionManifest produced by canonical builder. diff --git a/backend/src/services/clean_release/manifest_service.py b/backend/src/services/clean_release/manifest_service.py index 91e06245..500a3335 100644 --- a/backend/src/services/clean_release/manifest_service.py +++ b/backend/src/services/clean_release/manifest_service.py @@ -1,4 +1,5 @@ # #region ManifestService [C:5] [TYPE Module] [SEMANTICS clean-release, manifest, verify, digest] +# @defgroup Services Module group. # @BRIEF Build immutable distribution manifests with deterministic digest and version increment. # @LAYER Domain # @RELATION DEPENDS_ON -> [RepositoryRelations] @@ -21,6 +22,7 @@ from .repository import CleanReleaseRepository # #region build_manifest_snapshot [TYPE Function] +# @ingroup Services # @BRIEF Create a new immutable manifest version for a candidate. # @PRE Candidate is prepared, artifacts are available, candidate_id is valid. # @POST Returns persisted DistributionManifest with monotonically incremented version. diff --git a/backend/src/services/clean_release/mappers.py b/backend/src/services/clean_release/mappers.py index 971c7731..298183a2 100644 --- a/backend/src/services/clean_release/mappers.py +++ b/backend/src/services/clean_release/mappers.py @@ -1,4 +1,5 @@ # #region clean_release_mappers [C:3] [TYPE Module] [SEMANTICS clean-release, mapper, dto, entity, transform] +# @defgroup Services Module group. # @BRIEF Map between domain entities (SQLAlchemy models) and DTOs. # @LAYER Application # @RELATION DEPENDS_ON -> clean_release_dto diff --git a/backend/src/services/clean_release/policy_engine.py b/backend/src/services/clean_release/policy_engine.py index 0d4fc734..50b3c7c9 100644 --- a/backend/src/services/clean_release/policy_engine.py +++ b/backend/src/services/clean_release/policy_engine.py @@ -1,4 +1,5 @@ # #region PolicyEngine [C:5] [TYPE Module] [SEMANTICS clean-release, policy, validate, profile, artifact] +# @defgroup Services Module group. # @BRIEF Evaluate artifact/source policies for enterprise clean profile with deterministic outcomes. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] @@ -36,6 +37,7 @@ class SourceValidationResult: # #region CleanPolicyEngine [TYPE Class] +# @defgroup Services Module group. # @PRE Active policy exists and is internally consistent. # @POST Deterministic classification and source validation are available. # @TEST_CONTRACT CandidateEvaluationInput -> PolicyValidationResult|SourceValidationResult diff --git a/backend/src/services/clean_release/policy_resolution_service.py b/backend/src/services/clean_release/policy_resolution_service.py index f6a89b29..6e6272d4 100644 --- a/backend/src/services/clean_release/policy_resolution_service.py +++ b/backend/src/services/clean_release/policy_resolution_service.py @@ -1,4 +1,5 @@ # #region PolicyResolutionService [C:5] [TYPE Module] [SEMANTICS clean-release, policy, resolution, registry] +# @defgroup Services Module group. # @BRIEF Resolve trusted policy and registry snapshots from ConfigManager without runtime overrides. # @LAYER Domain # @RELATION DEPENDS_ON -> [ConfigManager] @@ -18,6 +19,7 @@ from .repository import CleanReleaseRepository # #region resolve_trusted_policy_snapshots [TYPE Function] +# @ingroup Services # @BRIEF Resolve immutable trusted policy and registry snapshots using active config IDs only. # @PRE ConfigManager provides active_policy_id and active_registry_id; repository contains referenced snapshots. # @POST Returns immutable policy and registry snapshots; runtime override attempts are rejected. diff --git a/backend/src/services/clean_release/preparation_service.py b/backend/src/services/clean_release/preparation_service.py index a6e49325..68204b18 100644 --- a/backend/src/services/clean_release/preparation_service.py +++ b/backend/src/services/clean_release/preparation_service.py @@ -1,4 +1,5 @@ # #region PreparationService [C:5] [TYPE Module] [SEMANTICS clean-release, prepare, validate, policy, manifest] +# @defgroup Services Module group. # @BRIEF Prepare release candidate by policy evaluation and deterministic manifest creation. # @LAYER Domain # @RELATION DEPENDS_ON -> [PolicyEngine] @@ -90,6 +91,7 @@ def prepare_candidate( # #region prepare_candidate_legacy [TYPE Function] +# @ingroup Services # @BRIEF Legacy compatibility wrapper kept for migration period. # @PRE Same as prepare_candidate. # @POST Delegates to canonical prepare_candidate and preserves response shape. diff --git a/backend/src/services/clean_release/publication_service.py b/backend/src/services/clean_release/publication_service.py index 5a3a326e..b86dc7c5 100644 --- a/backend/src/services/clean_release/publication_service.py +++ b/backend/src/services/clean_release/publication_service.py @@ -1,4 +1,5 @@ # #region PublicationService [C:5] [TYPE Module] [SEMANTICS clean-release, publication, publish, release] +# @defgroup Services Module group. # @BRIEF Enforce publication and revocation gates with append-only publication records. # @LAYER Domain # @RELATION DEPENDS_ON -> [RepositoryRelations] @@ -84,6 +85,7 @@ def _latest_approval_for_candidate( # #region publish_candidate [TYPE Function] +# @ingroup Services # @BRIEF Create immutable publication record for approved candidate. # @PRE Candidate exists, report belongs to candidate, latest approval is APPROVED. # @POST New ACTIVE publication record is appended. @@ -164,6 +166,7 @@ def publish_candidate( # #region revoke_publication [TYPE Function] +# @ingroup Services # @BRIEF Revoke existing publication record without deleting history. # @PRE publication_id exists in repository publication store. # @POST Target publication status becomes REVOKED and updated record is returned. diff --git a/backend/src/services/clean_release/report_builder.py b/backend/src/services/clean_release/report_builder.py index 2d1329e9..43763a9c 100644 --- a/backend/src/services/clean_release/report_builder.py +++ b/backend/src/services/clean_release/report_builder.py @@ -1,4 +1,5 @@ # #region ReportBuilder [C:5] [TYPE Module] [SEMANTICS report, clean-release, compliance, builder, render] +# @defgroup Services Module group. # @BRIEF Build and persist compliance reports with consistent counter invariants. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] diff --git a/backend/src/services/clean_release/repositories/__init__.py b/backend/src/services/clean_release/repositories/__init__.py index e1b81522..e731d848 100644 --- a/backend/src/services/clean_release/repositories/__init__.py +++ b/backend/src/services/clean_release/repositories/__init__.py @@ -1,4 +1,5 @@ # #region clean_release_repositories [C:3] [TYPE Module] [SEMANTICS clean-release, repository, package, export] +# @defgroup Services Module group. # @BRIEF Export all clean release repositories. # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/approval_repository.py b/backend/src/services/clean_release/repositories/approval_repository.py index ccaddd76..520c45a9 100644 --- a/backend/src/services/clean_release/repositories/approval_repository.py +++ b/backend/src/services/clean_release/repositories/approval_repository.py @@ -1,4 +1,5 @@ # #region approval_repository [C:3] [TYPE Module] [SEMANTICS clean-release, approval, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query approval decisions. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/artifact_repository.py b/backend/src/services/clean_release/repositories/artifact_repository.py index a4533123..3fced3dc 100644 --- a/backend/src/services/clean_release/repositories/artifact_repository.py +++ b/backend/src/services/clean_release/repositories/artifact_repository.py @@ -1,4 +1,5 @@ # #region artifact_repository [C:3] [TYPE Module] [SEMANTICS clean-release, artifact, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query candidate artifacts. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/audit_repository.py b/backend/src/services/clean_release/repositories/audit_repository.py index 2e80fb07..1212dea4 100644 --- a/backend/src/services/clean_release/repositories/audit_repository.py +++ b/backend/src/services/clean_release/repositories/audit_repository.py @@ -1,4 +1,5 @@ # #region audit_repository [C:3] [TYPE Module] [SEMANTICS clean-release, audit, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query audit logs for clean release operations. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/candidate_repository.py b/backend/src/services/clean_release/repositories/candidate_repository.py index e9eb46e9..8f3fc093 100644 --- a/backend/src/services/clean_release/repositories/candidate_repository.py +++ b/backend/src/services/clean_release/repositories/candidate_repository.py @@ -1,4 +1,5 @@ # #region candidate_repository [C:3] [TYPE Module] [SEMANTICS clean-release, candidate, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query release candidates. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/compliance_repository.py b/backend/src/services/clean_release/repositories/compliance_repository.py index 74babf22..ec996845 100644 --- a/backend/src/services/clean_release/repositories/compliance_repository.py +++ b/backend/src/services/clean_release/repositories/compliance_repository.py @@ -1,4 +1,5 @@ # #region compliance_repository [C:3] [TYPE Module] [SEMANTICS clean-release, compliance, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query compliance runs, stage runs, and violations. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/manifest_repository.py b/backend/src/services/clean_release/repositories/manifest_repository.py index 34237330..debc4492 100644 --- a/backend/src/services/clean_release/repositories/manifest_repository.py +++ b/backend/src/services/clean_release/repositories/manifest_repository.py @@ -1,4 +1,5 @@ # #region ManifestRepositoryModule [C:3] [TYPE Module] [SEMANTICS clean-release, manifest, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query distribution manifests. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> DistributionManifest @@ -13,6 +14,7 @@ from src.models.clean_release import DistributionManifest # #region ManifestRepository [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Encapsulates database CRUD operations for DistributionManifest entities. # @RELATION DEPENDS_ON -> DistributionManifest # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy.Session] diff --git a/backend/src/services/clean_release/repositories/policy_repository.py b/backend/src/services/clean_release/repositories/policy_repository.py index 1c9b5fd4..42148a46 100644 --- a/backend/src/services/clean_release/repositories/policy_repository.py +++ b/backend/src/services/clean_release/repositories/policy_repository.py @@ -1,4 +1,5 @@ # #region policy_repository [C:3] [TYPE Module] [SEMANTICS clean-release, policy, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query policy and registry snapshots. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/publication_repository.py b/backend/src/services/clean_release/repositories/publication_repository.py index 5fd0c47a..2c272dd7 100644 --- a/backend/src/services/clean_release/repositories/publication_repository.py +++ b/backend/src/services/clean_release/repositories/publication_repository.py @@ -1,4 +1,5 @@ # #region publication_repository [C:3] [TYPE Module] [SEMANTICS clean-release, publication, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query publication records. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repositories/report_repository.py b/backend/src/services/clean_release/repositories/report_repository.py index 1d49eec5..713cb38f 100644 --- a/backend/src/services/clean_release/repositories/report_repository.py +++ b/backend/src/services/clean_release/repositories/report_repository.py @@ -1,4 +1,5 @@ # #region report_repository [C:3] [TYPE Module] [SEMANTICS clean-release, report, repository, persistence, crud] +# @defgroup Services Module group. # @BRIEF Persist and query compliance reports. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [EXT:Library:sqlalchemy] diff --git a/backend/src/services/clean_release/repository.py b/backend/src/services/clean_release/repository.py index 26ecc2c0..192d236e 100644 --- a/backend/src/services/clean_release/repository.py +++ b/backend/src/services/clean_release/repository.py @@ -1,4 +1,5 @@ # #region RepositoryRelations [C:5] [TYPE Module] [SEMANTICS pydantic, clean-release, repository, relations, release] +# @defgroup Services Module group. # @BRIEF Provide repository adapter for clean release entities with deterministic access methods. # @LAYER Infrastructure # @RELATION DEPENDS_ON -> [CleanReleaseModels] @@ -26,6 +27,7 @@ from ...models.clean_release import ( # #region CleanReleaseRepository [TYPE Class] +# @defgroup Services Module group. # @BRIEF Data access object for clean release lifecycle. @dataclass class CleanReleaseRepository: diff --git a/backend/src/services/clean_release/source_isolation.py b/backend/src/services/clean_release/source_isolation.py index 2eb79350..31ac5333 100644 --- a/backend/src/services/clean_release/source_isolation.py +++ b/backend/src/services/clean_release/source_isolation.py @@ -1,4 +1,5 @@ # #region SourceIsolation [C:5] [TYPE Module] [SEMANTICS clean-release, source, isolation, validate, resource] +# @defgroup Services Module group. # @BRIEF Validate that all resource endpoints belong to the approved internal source registry. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] diff --git a/backend/src/services/clean_release/stages/__init__.py b/backend/src/services/clean_release/stages/__init__.py index 2ceedf23..94d03c50 100644 --- a/backend/src/services/clean_release/stages/__init__.py +++ b/backend/src/services/clean_release/stages/__init__.py @@ -1,4 +1,5 @@ # #region ComplianceStages [C:5] [TYPE Module] [SEMANTICS clean-release, compliance, stage, package] +# @defgroup Services Module group. # @BRIEF Define compliance stage order and helper functions for deterministic run-state evaluation. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] @@ -34,6 +35,7 @@ MANDATORY_STAGE_ORDER: list[ComplianceStageName] = [ # #region build_default_stages [TYPE Function] +# @ingroup Services # @BRIEF Build default deterministic stage pipeline implementation order. # @PRE None. # @POST Returns stage instances in mandatory execution order. @@ -50,6 +52,7 @@ def build_default_stages() -> list[ComplianceStage]: # #region stage_result_map [TYPE Function] +# @ingroup Services # @BRIEF Convert stage result list to dictionary by stage name. # @PRE stage_results may be empty or contain unique stage names. # @POST Returns stage->status dictionary for downstream evaluation. @@ -89,6 +92,7 @@ def stage_result_map( # #region missing_mandatory_stages [TYPE Function] +# @ingroup Services # @BRIEF Identify mandatory stages that are absent from run results. # @PRE stage_status_map contains zero or more known stage statuses. # @POST Returns ordered list of missing mandatory stages. @@ -102,6 +106,7 @@ def missing_mandatory_stages( # #region derive_final_status [TYPE Function] +# @ingroup Services # @BRIEF Derive final run status from stage results with deterministic blocking behavior. # @PRE Stage statuses correspond to compliance checks. # @POST Returns one of PASSED/BLOCKED/ERROR according to mandatory stage outcomes. diff --git a/backend/src/services/clean_release/stages/base.py b/backend/src/services/clean_release/stages/base.py index 49cb7a17..93494781 100644 --- a/backend/src/services/clean_release/stages/base.py +++ b/backend/src/services/clean_release/stages/base.py @@ -1,4 +1,5 @@ # #region ComplianceStageBase [C:5] [TYPE Module] [SEMANTICS pydantic, clean-release, stage, compliance, context] +# @defgroup Services Module group. # @BRIEF Define shared contracts and helpers for pluggable clean-release compliance stages. # @LAYER Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] @@ -29,6 +30,7 @@ from ..enums import ComplianceStageName, ViolationSeverity # #region ComplianceStageContext [TYPE Class] +# @defgroup Services Module group. # @BRIEF Immutable input envelope passed to each compliance stage. @dataclass(frozen=True) class ComplianceStageContext: @@ -43,6 +45,7 @@ class ComplianceStageContext: # #region StageExecutionResult [TYPE Class] +# @defgroup Services Module group. # @BRIEF Structured stage output containing decision, details and violations. @dataclass class StageExecutionResult: @@ -55,6 +58,7 @@ class StageExecutionResult: # #region ComplianceStage [TYPE Class] +# @defgroup Services Module group. # @BRIEF Protocol for pluggable stage implementations. class ComplianceStage(Protocol): stage_name: ComplianceStageName @@ -66,6 +70,7 @@ class ComplianceStage(Protocol): # #region build_stage_run_record [TYPE Function] +# @ingroup Services # @BRIEF Build persisted stage run record from stage result. # @PRE run_id and stage_name are non-empty. # @POST Returns ComplianceStageRun with deterministic identifiers and timestamps. @@ -97,6 +102,7 @@ def build_stage_run_record( # #region build_violation [TYPE Function] +# @ingroup Services # @BRIEF Construct a compliance violation with normalized defaults. # @PRE run_id, stage_name, code and message are non-empty. # @POST Returns immutable-style violation payload ready for persistence. diff --git a/backend/src/services/clean_release/stages/data_purity.py b/backend/src/services/clean_release/stages/data_purity.py index 7cf4d523..51245281 100644 --- a/backend/src/services/clean_release/stages/data_purity.py +++ b/backend/src/services/clean_release/stages/data_purity.py @@ -1,4 +1,5 @@ # #region data_purity [C:5] [TYPE Module] [SEMANTICS clean-release, data, purity, validate, manifest] +# @defgroup Services Module group. # @BRIEF Evaluate manifest purity counters and emit blocking violations for prohibited artifacts. # @LAYER Domain # @RELATION IMPLEMENTS -> [ComplianceStage] @@ -15,6 +16,7 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region DataPurityStage [TYPE Class] +# @defgroup Services Module group. # @BRIEF Validate manifest summary for prohibited artifacts. # @PRE context.manifest.content_json contains summary block or defaults to safe counters. # @POST Returns PASSED when no prohibited artifacts are detected, otherwise BLOCKED with violations. diff --git a/backend/src/services/clean_release/stages/internal_sources_only.py b/backend/src/services/clean_release/stages/internal_sources_only.py index 38a5e2b9..5fbafb72 100644 --- a/backend/src/services/clean_release/stages/internal_sources_only.py +++ b/backend/src/services/clean_release/stages/internal_sources_only.py @@ -1,4 +1,5 @@ # #region internal_sources_only [C:5] [TYPE Module] [SEMANTICS clean-release, internal, source, validate] +# @defgroup Services Module group. # @BRIEF Verify manifest-declared sources belong to trusted internal registry allowlist. # @LAYER Domain # @RELATION IMPLEMENTS -> [ComplianceStage] @@ -15,6 +16,7 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region InternalSourcesOnlyStage [TYPE Class] +# @defgroup Services Module group. # @BRIEF Enforce internal-source-only policy from trusted registry snapshot. # @PRE context.registry.allowed_hosts is available. # @POST Returns PASSED when all hosts are allowed; otherwise BLOCKED and violations captured. diff --git a/backend/src/services/clean_release/stages/manifest_consistency.py b/backend/src/services/clean_release/stages/manifest_consistency.py index 12472d31..89097acd 100644 --- a/backend/src/services/clean_release/stages/manifest_consistency.py +++ b/backend/src/services/clean_release/stages/manifest_consistency.py @@ -1,4 +1,5 @@ # #region manifest_consistency [C:5] [TYPE Module] [SEMANTICS clean-release, manifest, consistency, validate] +# @defgroup Services Module group. # @BRIEF Ensure run is bound to the exact manifest snapshot and digest used at run creation time. # @LAYER Domain # @RELATION IMPLEMENTS -> [ComplianceStage] @@ -15,6 +16,7 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region ManifestConsistencyStage [TYPE Class] +# @defgroup Services Module group. # @BRIEF Validate run/manifest linkage consistency. # @PRE context.run and context.manifest are loaded from repository for same run. # @POST Returns PASSED when digests match, otherwise ERROR with one violation. diff --git a/backend/src/services/clean_release/stages/no_external_endpoints.py b/backend/src/services/clean_release/stages/no_external_endpoints.py index 61904d92..a91121e8 100644 --- a/backend/src/services/clean_release/stages/no_external_endpoints.py +++ b/backend/src/services/clean_release/stages/no_external_endpoints.py @@ -1,4 +1,5 @@ # #region no_external_endpoints [C:5] [TYPE Module] [SEMANTICS clean-release, endpoint, validate, manifest, compliance] +# @defgroup Services Module group. # @BRIEF Block manifest payloads that expose external endpoints outside trusted schemes and hosts. # @LAYER Domain # @RELATION IMPLEMENTS -> [ComplianceStage] @@ -17,6 +18,7 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region NoExternalEndpointsStage [TYPE Class] +# @defgroup Services Module group. # @BRIEF Validate endpoint references from manifest against trusted registry. # @PRE context.registry includes allowed hosts and schemes. # @POST Returns PASSED when all endpoints are trusted, otherwise BLOCKED with endpoint violations. diff --git a/backend/src/services/dataset_review/__init__.py b/backend/src/services/dataset_review/__init__.py index 2f7ed48b..1dc3688b 100644 --- a/backend/src/services/dataset_review/__init__.py +++ b/backend/src/services/dataset_review/__init__.py @@ -1,4 +1,5 @@ # #region dataset_review [TYPE Module] [SEMANTICS dataset, review, orchestration, package] +# @defgroup DatasetReview Module group. # # @BRIEF Provides services for dataset-centered orchestration flow. # @RELATION CALLS -> [DatasetReviewOrchestrator] diff --git a/backend/src/services/dataset_review/clarification_engine.py b/backend/src/services/dataset_review/clarification_engine.py index 15e032ec..eb148c44 100644 --- a/backend/src/services/dataset_review/clarification_engine.py +++ b/backend/src/services/dataset_review/clarification_engine.py @@ -1,4 +1,5 @@ # #region ClarificationEngine [C:4] [TYPE Module] [SEMANTICS pydantic, dataset, clarification, finding, resolution] +# @defgroup DatasetReview Module group. # @BRIEF Manage one-question-at-a-time clarification state, deterministic answer persistence, and readiness/finding updates. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] @@ -51,6 +52,7 @@ from src.services.dataset_review.repositories.session_repository import ( # #region ClarificationQuestionPayload [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Typed active-question payload returned to the API layer. @dataclass class ClarificationQuestionPayload: @@ -69,6 +71,7 @@ class ClarificationQuestionPayload: # #region ClarificationStateResult [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Clarification state result carrying the current session, active payload, and changed findings. @dataclass class ClarificationStateResult: @@ -82,6 +85,7 @@ class ClarificationStateResult: # #region ClarificationAnswerCommand [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Typed answer command for clarification state mutation. @dataclass class ClarificationAnswerCommand: @@ -96,6 +100,7 @@ class ClarificationAnswerCommand: # #region ClarificationEngine [C:4] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Provide deterministic one-question-at-a-time clarification selection and answer persistence. # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] # @RELATION CALLS -> [ClarificationHelpers] diff --git a/backend/src/services/dataset_review/clarification_pkg/_helpers.py b/backend/src/services/dataset_review/clarification_pkg/_helpers.py index fc288ed3..4da2bd3a 100644 --- a/backend/src/services/dataset_review/clarification_pkg/_helpers.py +++ b/backend/src/services/dataset_review/clarification_pkg/_helpers.py @@ -1,4 +1,5 @@ # #region ClarificationHelpers [C:3] [TYPE Module] [SEMANTICS dataset, clarification, question, selection, normalization] +# @defgroup DatasetReview Module group. # @BRIEF Pure helper functions for clarification engine — question selection, counting, normalization, finding upsert, and readiness derivation. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] @@ -24,6 +25,7 @@ from src.models.dataset_review import ( # #region select_next_open_question [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Select the next unresolved question in deterministic priority order. def select_next_open_question( clarification_session: ClarificationSession, @@ -63,6 +65,7 @@ def count_remaining_questions(clarification_session: ClarificationSession) -> in # #region normalize_answer_value [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Validate and normalize answer payload based on answer kind and active question options. def normalize_answer_value( answer_kind: AnswerKind, @@ -104,6 +107,7 @@ def build_impact_summary( # #region upsert_clarification_finding [C:3] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Keep one finding per clarification topic aligned with answer outcome and unresolved visibility rules. # @RELATION DEPENDS_ON -> [ValidationFinding] def upsert_clarification_finding( @@ -171,6 +175,7 @@ def upsert_clarification_finding( # #region derive_readiness_state [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Recompute readiness after clarification mutation while preserving unresolved visibility semantics. def derive_readiness_state( session: DatasetReviewSession, @@ -189,6 +194,7 @@ def derive_readiness_state( # #region derive_recommended_action [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Recompute next-action guidance after clarification mutations. def derive_recommended_action( session: DatasetReviewSession, diff --git a/backend/src/services/dataset_review/event_logger.py b/backend/src/services/dataset_review/event_logger.py index b7c57e97..1c230465 100644 --- a/backend/src/services/dataset_review/event_logger.py +++ b/backend/src/services/dataset_review/event_logger.py @@ -1,4 +1,5 @@ # #region SessionEventLoggerModule [C:4] [TYPE Module] [SEMANTICS sqlalchemy, dataset, event, logging, session] +# @defgroup DatasetReview Module group. # @BRIEF Persist explicit session mutation events for dataset-review audit trails without weakening ownership or approval invariants. # @LAYER Domain # @RELATION DEPENDS_ON -> [SessionEvent] @@ -11,6 +12,7 @@ from __future__ import annotations # #region SessionEventLoggerImports [TYPE Block] +# @ingroup DatasetReview from dataclasses import dataclass, field from typing import Any @@ -23,6 +25,7 @@ from src.models.dataset_review import DatasetReviewSession, SessionEvent # #region SessionEventPayload [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Typed input contract for one persisted dataset-review session audit event. @dataclass(frozen=True) class SessionEventPayload: @@ -37,6 +40,7 @@ class SessionEventPayload: # #region SessionEventLogger [C:4] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Persist explicit dataset-review session audit events with meaningful runtime reasoning logs. # @RELATION DEPENDS_ON -> [SessionEvent] # @RELATION DEPENDS_ON -> [SessionEventPayload] diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index 13ba8977..ea7606d5 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -1,4 +1,5 @@ # #region DatasetReviewOrchestrator [C:5] [TYPE Module] [SEMANTICS pydantic, dataset, review, orchestration, session] +# @defgroup DatasetReview Module group. # @BRIEF Coordinate dataset review session startup and lifecycle-safe intake recovery for one authenticated user. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] @@ -83,6 +84,7 @@ logger = cast(Any, logger) # #region DatasetReviewOrchestrator [C:5] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Coordinate safe session startup while preserving cross-user isolation and explicit partial recovery. # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] # @RELATION DEPENDS_ON -> [SupersetContextExtractor] diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_commands.py b/backend/src/services/dataset_review/orchestrator_pkg/_commands.py index 0480b3e6..70f6cac5 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_commands.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_commands.py @@ -1,4 +1,5 @@ # #region OrchestratorCommands [C:2] [TYPE Module] [SEMANTICS dataset, review, command, dataclass, boundary] +# @defgroup DatasetReview Module group. # @BRIEF Typed command and result dataclasses for dataset review orchestration boundary. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] @@ -19,6 +20,7 @@ from src.models.dataset_review import ( # #region StartSessionCommand [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Typed input contract for starting a dataset review session. @dataclass class StartSessionCommand: @@ -32,6 +34,7 @@ class StartSessionCommand: # #region StartSessionResult [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Session-start result carrying the persisted session and intake recovery metadata. @dataclass class StartSessionResult: @@ -44,6 +47,7 @@ class StartSessionResult: # #region PreparePreviewCommand [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Typed input contract for compiling one Superset-backed session preview. @dataclass class PreparePreviewCommand: @@ -56,6 +60,7 @@ class PreparePreviewCommand: # #region PreparePreviewResult [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Result contract for one persisted compiled preview attempt. @dataclass class PreparePreviewResult: @@ -68,6 +73,7 @@ class PreparePreviewResult: # #region LaunchDatasetCommand [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Typed input contract for launching one dataset-review session into SQL Lab. @dataclass class LaunchDatasetCommand: @@ -80,6 +86,7 @@ class LaunchDatasetCommand: # #region LaunchDatasetResult [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Launch result carrying immutable run context and any gate blockers. @dataclass class LaunchDatasetResult: diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py index 6af9dbf6..ba390e5a 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py @@ -1,4 +1,5 @@ # #region OrchestratorHelpers [C:4] [TYPE Module] [SEMANTICS dataset, review, snapshot, fingerprint, recovery] +# @defgroup DatasetReview Module group. # @BRIEF Pure helper methods extracted from DatasetReviewOrchestrator for INV_7 compliance — snapshot, blockers, fingerprint, recovery bootstrap. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] @@ -32,6 +33,7 @@ logger = cast(Any, logger) # #region parse_dataset_selection [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Normalize dataset-selection payload into canonical session references. def parse_dataset_selection(source_input: str) -> tuple[str, int | None]: normalized = str(source_input or "").strip() @@ -52,6 +54,7 @@ def parse_dataset_selection(source_input: str) -> tuple[str, int | None]: # #region build_initial_profile [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Create the first profile snapshot so exports and detail views remain usable immediately after intake. def build_initial_profile( session_id: str, @@ -94,6 +97,7 @@ def build_initial_profile( # #region build_partial_recovery_findings [C:3] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Project partial Superset intake recovery into explicit findings without blocking session usability. # @PRE parsed_context.partial_recovery is true. # @POST Returns warning-level findings that preserve usable but incomplete state. @@ -121,6 +125,7 @@ def build_partial_recovery_findings(parsed_context: Any) -> list[ValidationFindi # #region extract_effective_filter_value [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Separate normalized filter payload metadata from the user-facing effective filter value. def extract_effective_filter_value( normalized_value: Any, raw_value: Any @@ -137,6 +142,7 @@ def extract_effective_filter_value( # #region build_execution_snapshot [C:4] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Build effective filters, template params, approvals, and fingerprint for preview and launch gating. # @PRE Session aggregate includes imported filters, template variables, and current execution mappings. # @POST Returns deterministic execution snapshot for current session state without mutating persistence. @@ -266,6 +272,7 @@ def build_execution_snapshot(session: DatasetReviewSession) -> dict[str, Any]: # #region build_launch_blockers [C:3] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Enforce launch gates from findings, approvals, and current preview truth. # @PRE execution_snapshot was computed from current session state. # @POST Returns explicit blocker codes for every unmet launch invariant. @@ -306,6 +313,7 @@ def build_launch_blockers( # #region get_latest_preview [C:2] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Resolve the current latest preview snapshot for one session aggregate. def get_latest_preview(session: DatasetReviewSession) -> CompiledPreview | None: session_record = cast(Any, session) diff --git a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py index 0a298854..0927d659 100644 --- a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py +++ b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py @@ -1,3 +1,8 @@ +import os + +os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:") +os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing") + from pathlib import Path from datetime import UTC import pytest @@ -125,11 +130,11 @@ def test_bump_session_version_updates_last_activity(db_session): ) repo.create_session(session) - before_activity = session.last_activity_at - # Normalize to offset-aware for comparison (DB may store naive) - if before_activity is not None and before_activity.tzinfo is None: - before_activity = before_activity.replace(tzinfo=UTC) - next_version = repo.bump_session_version(session) + before_activity = session.last_activity_at + # Normalize to offset-aware for comparison (DB may store naive) + if before_activity is not None and before_activity.tzinfo is None: + before_activity = before_activity.replace(tzinfo=UTC) + next_version = repo.bump_session_version(session) assert next_version == 1 assert session.version == 1 diff --git a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py index ed855ec4..62c4302e 100644 --- a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py +++ b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py @@ -1,4 +1,5 @@ # #region SessionRepositoryMutations [C:4] [TYPE Module] [SEMANTICS dataset, repository, mutation, session, persistence] +# @defgroup DatasetReview Module group. # @BRIEF Persistence mutation operations for dataset review session aggregates — profile/findings, recovery state, preview, run context. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] @@ -29,6 +30,7 @@ logger = cast(Any, logger) # #region save_profile_and_findings [C:4] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Persist profile state and replace validation findings for an owned session in one transaction. # @PRE session_id belongs to user_id and the supplied profile/findings belong to the same aggregate scope. # @POST stored profile matches the current session and findings are replaced by the supplied collection. @@ -72,6 +74,7 @@ def save_profile_and_findings( # #region save_recovery_state [C:4] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Persist imported filters, template variables, and initial execution mappings for one owned session. # @PRE session_id belongs to user_id. # @POST Recovery state persisted to database. @@ -119,6 +122,7 @@ def save_recovery_state( # #region save_preview [C:3] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Persist a preview snapshot and mark prior session previews stale. # @PRE session_id belongs to user_id and preview is prepared for the same session aggregate. # @POST preview is persisted and the session points to the latest preview identifier. @@ -154,6 +158,7 @@ def save_preview( # #region save_run_context [C:3] [TYPE Function] +# @ingroup DatasetReview # @BRIEF Persist an immutable launch audit snapshot for an owned session. # @PRE session_id belongs to user_id and run_context targets the same aggregate. # @POST run context is persisted and linked as the latest launch snapshot for the session. diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index 5693364a..dfb05f00 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -1,4 +1,5 @@ # #region DatasetReviewSessionRepository [C:5] [TYPE Module] [SEMANTICS dataset, repository, session, aggregate, persistence] +# @defgroup DatasetReview Module group. # @BRIEF Persist and retrieve dataset review session aggregates, including readiness, findings, semantic decisions, clarification state, previews, and run contexts. # @LAYER Domain # @RELATION DEPENDS_ON -> [DatasetReviewSession] @@ -42,6 +43,7 @@ logger = cast(Any, logger) # #region DatasetReviewSessionVersionConflictError [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Signal optimistic-lock conflicts for dataset review session mutations. class DatasetReviewSessionVersionConflictError(ValueError): def __init__(self, session_id: str, expected_version: int, actual_version: int): @@ -57,6 +59,7 @@ class DatasetReviewSessionVersionConflictError(ValueError): # #region DatasetReviewSessionRepository [C:4] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Enforce ownership-scoped persistence and retrieval for dataset review session aggregates. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [SessionEventLogger] diff --git a/backend/src/services/dataset_review/semantic_resolver.py b/backend/src/services/dataset_review/semantic_resolver.py index 51c12a0c..627e94cb 100644 --- a/backend/src/services/dataset_review/semantic_resolver.py +++ b/backend/src/services/dataset_review/semantic_resolver.py @@ -1,4 +1,5 @@ # #region SemanticSourceResolver [C:4] [TYPE Module] [SEMANTICS pydantic, dataset, semantic, resolver, mapping] +# @defgroup DatasetReview Module group. # @BRIEF Resolve and rank semantic candidates from trusted dictionary-like sources before any inferred fallback. # @LAYER Domain # @RELATION DEPENDS_ON -> [LLMProviderService] @@ -15,6 +16,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping # #region imports [TYPE Block] +# @ingroup DatasetReview from dataclasses import dataclass, field from difflib import SequenceMatcher from typing import Any @@ -31,6 +33,7 @@ from src.models.dataset_review import ( # #region DictionaryResolutionResult [C:2] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Carries field-level dictionary resolution output with explicit review and partial-recovery state. @dataclass class DictionaryResolutionResult: @@ -42,6 +45,7 @@ class DictionaryResolutionResult: # #region SemanticSourceResolver [C:4] [TYPE Class] +# @defgroup DatasetReview Module group. # @BRIEF Resolve semantic candidates from trusted sources while preserving manual locks and confidence ordering. # @RELATION DEPENDS_ON -> [SemanticFieldEntry] # @RELATION DEPENDS_ON -> [SemanticCandidate] diff --git a/backend/src/services/git/__init__.py b/backend/src/services/git/__init__.py index 59ad5ef9..cdf35962 100644 --- a/backend/src/services/git/__init__.py +++ b/backend/src/services/git/__init__.py @@ -1,4 +1,5 @@ # #region GitServiceModule [C:3] [TYPE Module] [SEMANTICS git, package, export, mixin, decomposition] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Composed GitService via multiple inheritance from domain-specific mixins. # @RELATION DEPENDS_ON -> [GitServiceBase] @@ -30,6 +31,7 @@ __all__ = ["GitService"] # #region GitService [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Composed GitService class providing all Git operations — init, clone, branches, commits, push, pull, # merge, status, diff, history, and provider API operations (Gitea, GitHub, GitLab). # @RELATION INHERITS -> [GitServiceBase] diff --git a/backend/src/services/git/_base.py b/backend/src/services/git/_base.py index c4e847b5..2d16b520 100644 --- a/backend/src/services/git/_base.py +++ b/backend/src/services/git/_base.py @@ -1,4 +1,5 @@ # #region GitServiceBase [C:4] [TYPE Module] [SEMANTICS git, repository, clone, base, mixin, lock, http] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), identity configuration, concurrent locking, and shared HTTP client pool. # @RELATION DEPENDS_ON -> [GitRepository] @@ -29,6 +30,7 @@ from src.models.git import GitRepository # #region GitServiceBase [C:4] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Base class for GitService providing initialization, path resolution, repository lifecycle, identity, concurrent locking, and shared HTTP client. # @PRE base_path is a valid string path. # @POST GitService is initialized; base_path directory exists; shared AsyncClient ready; lock registry empty. diff --git a/backend/src/services/git/_branch.py b/backend/src/services/git/_branch.py index 964b98ad..a2e97297 100644 --- a/backend/src/services/git/_branch.py +++ b/backend/src/services/git/_branch.py @@ -1,4 +1,5 @@ # #region GitServiceBranchMixin [C:4] [TYPE Module] [SEMANTICS git, branch, checkout, list, namespace, lock] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes (all concurrent-safe via per-dashboard locks). # @RELATION CALLED_BY -> [GitService] @@ -14,6 +15,7 @@ from src.core.logger import belief_scope, logger # #region GitServiceBranchMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing branch and commit operations for GitService. class GitServiceBranchMixin: # region _ensure_gitflow_branches [C:4] [TYPE Function] [SEMANTICS git,gitflow,branch,lock] diff --git a/backend/src/services/git/_gitea.py b/backend/src/services/git/_gitea.py index 6d61d800..93e54d65 100644 --- a/backend/src/services/git/_gitea.py +++ b/backend/src/services/git/_gitea.py @@ -1,4 +1,5 @@ # #region GitServiceGiteaMixin [C:4] [TYPE Module] [SEMANTICS git, gitea, api, remote, connection, http_pool] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation. Uses shared self._http_client for connection pooling. # @RELATION CALLED_BY -> [GitService] @@ -13,6 +14,7 @@ from src.models.git import GitProvider # #region GitServiceGiteaMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing Gitea API operations for GitService. class GitServiceGiteaMixin: # region test_connection [TYPE Function] diff --git a/backend/src/services/git/_merge.py b/backend/src/services/git/_merge.py index 0ef84692..4aba6c36 100644 --- a/backend/src/services/git/_merge.py +++ b/backend/src/services/git/_merge.py @@ -1,4 +1,5 @@ # #region GitServiceMergeMixin [C:4] [TYPE Module] [SEMANTICS git, merge, branch, conflict, resolution, lock] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Merge operations for GitService — conflict detection, resolution, abort, continue, and direct promote (all concurrent-safe via per-dashboard locks). # @RELATION CALLED_BY -> [GitService] @@ -16,6 +17,7 @@ from src.core.logger import belief_scope, logger # #region GitServiceMergeMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing merge operations for GitService. class GitServiceMergeMixin: # region _read_blob_text [TYPE Function] diff --git a/backend/src/services/git/_remote_providers.py b/backend/src/services/git/_remote_providers.py index 82f65867..6a5130be 100644 --- a/backend/src/services/git/_remote_providers.py +++ b/backend/src/services/git/_remote_providers.py @@ -1,4 +1,5 @@ # #region GitServiceRemoteMixin [C:4] [TYPE Module] [SEMANTICS git, provider, github, remote, url, http_pool] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation. Uses shared self._http_client for connection pooling. # @RELATION CALLED_BY -> [GitService] @@ -12,6 +13,7 @@ from src.core.logger import logger # #region GitServiceGithubMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing GitHub API operations for GitService. class GitServiceGithubMixin: # region create_github_repository [TYPE Function] @@ -55,6 +57,7 @@ class GitServiceGithubMixin: # #endregion GitServiceGithubMixin # #region GitServiceGitlabMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing GitLab API operations for GitService. class GitServiceGitlabMixin: # region create_gitlab_repository [TYPE Function] diff --git a/backend/src/services/git/_status.py b/backend/src/services/git/_status.py index e14f90c9..e777a1b0 100644 --- a/backend/src/services/git/_status.py +++ b/backend/src/services/git/_status.py @@ -1,4 +1,5 @@ # #region GitServiceStatusMixin [C:4] [TYPE Module] [SEMANTICS git, status, diff, log, history, lock] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Status, diff, and commit history operations for GitService (all concurrent-safe via per-dashboard locks). # @RELATION CALLED_BY -> [GitService] @@ -9,6 +10,7 @@ from src.core.logger import belief_scope, logger # #region GitServiceStatusMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing repository status, diff, and commit history for GitService. class GitServiceStatusMixin: # region _parse_status_porcelain [TYPE Function] diff --git a/backend/src/services/git/_sync.py b/backend/src/services/git/_sync.py index 359c6c28..9da10cf6 100644 --- a/backend/src/services/git/_sync.py +++ b/backend/src/services/git/_sync.py @@ -1,4 +1,5 @@ # #region GitServiceSyncMixin [C:4] [TYPE Module] [SEMANTICS git, sync, push, pull, remote, lock] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF Push and pull operations for GitService with origin host auto-alignment (concurrent-safe). # @RELATION CALLED_BY -> [GitService] @@ -15,6 +16,7 @@ from src.models.git import GitRepository, GitServerConfig # #region GitServiceSyncMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Mixin providing push and pull operations with origin host alignment. class GitServiceSyncMixin: # region push_changes [C:4] [TYPE Function] [SEMANTICS git,push,lock] diff --git a/backend/src/services/git/_url.py b/backend/src/services/git/_url.py index e157e3d0..05024d21 100644 --- a/backend/src/services/git/_url.py +++ b/backend/src/services/git/_url.py @@ -1,4 +1,5 @@ # #region GitServiceUrlMixin [C:3] [TYPE Module] [SEMANTICS git, url, parse, remote, endpoint] +# @defgroup Services Module group. # @LAYER Infrastructure # @BRIEF URL helper mixin for GitService — parse, normalize, align, and strip credentials from Git URLs. # @RELATION CALLED_BY -> [GitServiceSyncMixin] @@ -15,6 +16,7 @@ from src.models.git import GitRepository # #region GitServiceUrlMixin [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL. class GitServiceUrlMixin: # region _extract_http_host [TYPE Function] diff --git a/backend/src/services/health_service.py b/backend/src/services/health_service.py index 0cd0cf68..44b9bd0b 100644 --- a/backend/src/services/health_service.py +++ b/backend/src/services/health_service.py @@ -1,4 +1,5 @@ # #region health_service [C:3] [TYPE Module] [SEMANTICS sqlalchemy, health, dashboard, validation, aggregate] +# @defgroup Services Module group. # @BRIEF Business logic for aggregating dashboard health status from validation records. # @LAYER Service # @RELATION DEPENDS_ON -> [ValidationRecord] @@ -28,6 +29,7 @@ def _empty_dashboard_meta() -> dict[str, str | None]: # #region HealthService [C:4] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Aggregate latest dashboard validation state and manage persisted health report lifecycle. # @PRE Service is constructed with a live SQLAlchemy session and optional config manager. # @POST Exposes health summary aggregation and validation report deletion operations. diff --git a/backend/src/services/llm_prompt_templates.py b/backend/src/services/llm_prompt_templates.py index 1b624f7a..b10b87fc 100644 --- a/backend/src/services/llm_prompt_templates.py +++ b/backend/src/services/llm_prompt_templates.py @@ -1,4 +1,5 @@ # #region llm_prompt_templates [C:2] [TYPE Module] [SEMANTICS llm, prompt, template, normalization] +# @defgroup Services Module group. # @BRIEF Provide default LLM prompt templates and normalization helpers for runtime usage. # @LAYER Domain # @RELATION DEPENDS_ON -> [ConfigManager] @@ -13,6 +14,7 @@ import warnings from ..core.logger import logger # #region DEFAULT_LLM_PROMPTS [C:2] [TYPE Constant] +# @ingroup Services # @BRIEF Default prompt templates used by documentation, dashboard validation, and git commit generation. DEFAULT_LLM_PROMPTS: dict[str, str] = { "dashboard_validation_prompt": ( # Legacy v1 — deprecated for new tasks, kept for backward compat @@ -107,6 +109,7 @@ DEFAULT_LLM_PROMPTS: dict[str, str] = { # #region DEFAULT_LLM_PROVIDER_BINDINGS [C:2] [TYPE Constant] +# @ingroup Services # @BRIEF Default provider binding per task domain. DEFAULT_LLM_PROVIDER_BINDINGS: dict[str, str] = { "documentation": "", @@ -116,6 +119,7 @@ DEFAULT_LLM_PROVIDER_BINDINGS: dict[str, str] = { # #region DEFAULT_LLM_ASSISTANT_SETTINGS [C:2] [TYPE Constant] +# @ingroup Services # @BRIEF Default planner settings for assistant chat intent model/provider resolution. DEFAULT_LLM_ASSISTANT_SETTINGS: dict[str, str] = { "assistant_planner_provider": "", @@ -125,6 +129,7 @@ DEFAULT_LLM_ASSISTANT_SETTINGS: dict[str, str] = { # #region normalize_llm_settings [C:3] [TYPE Function] +# @ingroup Services # @BRIEF Ensure llm settings contain stable schema with prompts section and default templates. # @PRE llm_settings is dictionary-like value or None. # @POST Returned dict contains prompts with all required template keys. @@ -168,6 +173,7 @@ def normalize_llm_settings(llm_settings: Any) -> dict[str, Any]: # #endregion normalize_llm_settings # #region is_multimodal_model [C:3] [TYPE Function] +# @ingroup Services # @BRIEF Heuristically determine whether model supports image input required for dashboard validation. # @DEPRECATED Use the explicit `db_provider.is_multimodal` flag instead (see migration 9f8e7d6c5b4a). # @RATIONALE Added import warnings + warnings.warn(DeprecationWarning) to is_multimodal_model as a deprecation shim. @@ -219,6 +225,7 @@ def is_multimodal_model(model_name: str, provider_type: str | None = None) -> bo # #region resolve_bound_provider_id [C:3] [TYPE Function] +# @ingroup Services # @BRIEF Resolve provider id configured for a task binding with fallback to default provider. # @PRE llm_settings is normalized or raw dict from config. # @POST Returns configured provider id or fallback id/empty string when not defined. @@ -235,6 +242,7 @@ def resolve_bound_provider_id(llm_settings: Any, task_key: str) -> str: # #region render_prompt [C:3] [TYPE Function] +# @ingroup Services # @BRIEF Render prompt template using deterministic placeholder replacement with graceful fallback. # @PRE template is a string and variables values are already stringifiable. # @POST Returns rendered prompt text with known placeholders substituted. Warns about unfilled placeholders. diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index 4f91b34d..55445b9c 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -1,4 +1,5 @@ # #region llm_provider [C:3] [TYPE Module] [SEMANTICS sqlalchemy, llm, provider, encryption, config] +# @defgroup Services Module group. # @BRIEF Service for managing LLM provider configurations with encrypted API keys. # @LAYER Domain # @RELATION DEPENDS_ON -> [LLMProvider] @@ -22,6 +23,7 @@ MASKED_API_KEY_PLACEHOLDER = "********" # #region mask_api_key [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Mask an API key for safe display, showing first 4 and last 4 characters. # @PRE api_key is a plaintext string or None. # @POST Returns "****" for very short keys; "{first 2}...{last 2}" for <=8 chars; @@ -40,6 +42,7 @@ def mask_api_key(api_key: str | None) -> str: # #region is_masked_or_placeholder [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Predicate: True when api_key is None, empty, "********", or contains "...". # @PRE api_key can be None. # @POST Returns True only for non-real-key values. @@ -56,6 +59,7 @@ def is_masked_or_placeholder(api_key: str | None) -> bool: # #region LLMProviderService [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Service to manage LLM provider lifecycle. # @RELATION DEPENDS_ON -> [LLMProvider] # @RELATION DEPENDS_ON -> [EncryptionManager] diff --git a/backend/src/services/maintenance/_banner_renderer.py b/backend/src/services/maintenance/_banner_renderer.py index 70d4cd2f..c8603651 100644 --- a/backend/src/services/maintenance/_banner_renderer.py +++ b/backend/src/services/maintenance/_banner_renderer.py @@ -1,4 +1,5 @@ # #region MaintenanceBannerRenderer [C:3] [TYPE Module] [SEMANTICS maintenance, banner, renderer, text] +# @defgroup Services Module group. # @BRIEF Banner text rendering and rebuild helpers. Build aggregated markdown from # active maintenance events, format with template, apply HTML escaping. # @LAYER Service @@ -26,6 +27,7 @@ from ...models.maintenance import ( # #region build_banner_text [C:3] [TYPE Function] [SEMANTICS maintenance, banner, text, template, rendering] +# @ingroup Services # @BRIEF Build aggregated banner markdown text from events using settings template. # Single event: substitute {start_time}, {end_time}, {message} directly. # Multiple events: render each event as sub-block, combine as {message}. @@ -163,6 +165,7 @@ def _build_banner_text_for_dashboard( # #region rebuild_banner [C:3] [TYPE Function] [SEMANTICS maintenance, banner, rebuild, text, update] +# @ingroup Services # @BRIEF Rebuild aggregated banner text for a banner and update the Superset chart. # Query all active MaintenanceDashboardState records linked to banner, # build aggregated text via build_banner_text, update markdown chart in Superset. diff --git a/backend/src/services/maintenance/_chart_manager.py b/backend/src/services/maintenance/_chart_manager.py index 846a41a6..4a2fb3f0 100644 --- a/backend/src/services/maintenance/_chart_manager.py +++ b/backend/src/services/maintenance/_chart_manager.py @@ -1,4 +1,5 @@ # #region MaintenanceChartManager [C:3] [TYPE Module] [SEMANTICS maintenance, chart, banner, lifecycle] +# @defgroup Services Module group. # @BRIEF Chart operations for maintenance banners: create/get banner charts, process # per-dashboard state transitions for start and end orchestrators. # @LAYER Service @@ -22,6 +23,7 @@ from ._dashboard_scanner import _resolve_dashboard_title # #region ensure_banner_chart [C:3] [TYPE Function] [SEMANTICS maintenance, banner, chart, creation] +# @ingroup Services # @BRIEF Get or create a MaintenanceDashboardBanner for a dashboard in the target environment. # If an active banner exists in DB, return it. Otherwise, create a markdown chart in Superset, # update dashboard layout, and store the banner row. diff --git a/backend/src/services/maintenance/_dashboard_scanner.py b/backend/src/services/maintenance/_dashboard_scanner.py index 85addc53..186524a7 100644 --- a/backend/src/services/maintenance/_dashboard_scanner.py +++ b/backend/src/services/maintenance/_dashboard_scanner.py @@ -1,4 +1,5 @@ # #region MaintenanceDashboardScanner [C:3] [TYPE Module] [SEMANTICS maintenance, dashboard, scanner, tables] +# @defgroup Services Module group. # @BRIEF Dashboard discovery and filtering helpers for maintenance banner feature. # Scan Superset datasets, match tables, apply scope/excluded/forced filters. # @LAYER Service @@ -15,6 +16,7 @@ from ..sql_table_extractor import extract_tables_from_sql # #region find_affected_dashboards [C:3] [TYPE Function] [SEMANTICS maintenance, dashboard, discovery, tables] +# @ingroup Services # @BRIEF Find all dashboard IDs whose datasets reference any of the given tables. # Supports both table-based (schema, table_name) and virtual SQL datasets (via SqlTableExtractor). # Applies scope/excluded/forced filtering from MaintenanceSettings. diff --git a/backend/src/services/maintenance/_orchestrators.py b/backend/src/services/maintenance/_orchestrators.py index 0ed79e48..787dcae9 100644 --- a/backend/src/services/maintenance/_orchestrators.py +++ b/backend/src/services/maintenance/_orchestrators.py @@ -1,4 +1,5 @@ # #region MaintenanceOrchestrators [C:4] [TYPE Module] [SEMANTICS maintenance, orchestrator, lifecycle] +# @defgroup Services Module group. # @BRIEF C4 orchestrators for maintenance event lifecycle: start, end, end-all. # Private helpers extracted to sub-modules. All orchestrators use belief runtime. # @LAYER Service @@ -28,6 +29,7 @@ from ._dashboard_scanner import find_affected_dashboards # #region build_idempotency_key [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Build idempotency key from (tables, start_time, end_time). Sorted lowercased tables, ISO timestamps. # @PRE tables is a list of strings. start_time is a datetime or None. end_time is a datetime or None. # @POST Returns a tuple of (frozenset(str), str|None, str|None) for DB comparison. @@ -45,6 +47,7 @@ def build_idempotency_key( # #region start_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, start, orchestration, async] +# @ingroup Services # @BRIEF C4 orchestrator for starting a maintenance event. Runs inside a TaskManager task. # Idempotency check with SELECT ... FOR UPDATE. Creates event states, finds dashboards, # ensures banner charts, builds text, updates charts. Event transitions to ACTIVE or PARTIAL. @@ -181,6 +184,7 @@ async def start_maintenance( # #region end_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, end, orchestration, async, removal] +# @ingroup Services # @BRIEF C4 orchestrator for ending a maintenance event. Runs inside a TaskManager task. # Event → ENDING. Delegates per-state processing to _process_states_for_end. # Event → COMPLETED when done. @@ -258,6 +262,7 @@ async def end_maintenance( # #region end_all_maintenance [C:4] [TYPE Function] [SEMANTICS maintenance, end-all, orchestration, bulk] +# @ingroup Services # @BRIEF End ALL active maintenance events. Iterates active events, calls end_maintenance for each. # @PRE None. # @POST All events ended. All banners removed or rebuilt (last event removed). diff --git a/backend/src/services/mapping_service.py b/backend/src/services/mapping_service.py index aec499ca..bcf66df3 100644 --- a/backend/src/services/mapping_service.py +++ b/backend/src/services/mapping_service.py @@ -1,4 +1,5 @@ # #region mapping_service [C:5] [TYPE Module] [SEMANTICS mapping, database, superset, fuzzy, suggestion] +# @defgroup Services Module group. # # @BRIEF Orchestrates database fetching and fuzzy matching suggestions. # @LAYER Service @@ -18,6 +19,7 @@ from ..core.utils.matching import suggest_mappings # #region MappingService [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Service for handling database mapping logic. # @PRE config_manager exposes get_environments() with environment objects containing id. # @POST Provides client resolution and mapping suggestion methods. diff --git a/backend/src/services/notifications/providers.py b/backend/src/services/notifications/providers.py index cfb05897..d5f46421 100644 --- a/backend/src/services/notifications/providers.py +++ b/backend/src/services/notifications/providers.py @@ -1,4 +1,5 @@ # #region providers [C:5] [TYPE Module] [SEMANTICS notification, provider, smtp, telegram, slack] +# @defgroup Services Module group. # # @BRIEF Defines abstract base and concrete implementations for external notification delivery. # @RELATION CALLED_BY -> [NotificationService] @@ -39,6 +40,7 @@ async def _get_http_client() -> httpx.AsyncClient: # #region NotificationProvider [C:2] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Abstract base class for all notification providers. # @RELATION CALLED_BY -> [SMTPProvider] # @RELATION CALLED_BY -> [TelegramProvider] @@ -67,6 +69,7 @@ class NotificationProvider(ABC): # #region SMTPProvider [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Delivers notifications via SMTP. # @RELATION INHERITS -> [NotificationProvider] class SMTPProvider(NotificationProvider): @@ -110,6 +113,7 @@ class SMTPProvider(NotificationProvider): # #region TelegramProvider [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Delivers notifications via Telegram Bot API. # @RELATION INHERITS -> [NotificationProvider] class TelegramProvider(NotificationProvider): @@ -150,6 +154,7 @@ class TelegramProvider(NotificationProvider): # #region SlackProvider [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Delivers notifications via Slack Webhooks or API. # @RELATION INHERITS -> [NotificationProvider] class SlackProvider(NotificationProvider): diff --git a/backend/src/services/notifications/service.py b/backend/src/services/notifications/service.py index 2fe69510..dfb2a9a0 100644 --- a/backend/src/services/notifications/service.py +++ b/backend/src/services/notifications/service.py @@ -1,4 +1,5 @@ # #region service [C:5] [TYPE Module] [SEMANTICS fastapi, notification, dispatch, policy, routing] +# @defgroup Services Module group. # # @BRIEF Orchestrates notification routing based on user preferences and policy context. # @LAYER Domain @@ -35,6 +36,7 @@ from .providers import ( # #region NotificationService [C:4] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Routes validation reports to appropriate users and channels. # @RELATION DEPENDS_ON -> [NotificationProvider] # @RELATION DEPENDS_ON -> [ValidationRecord] diff --git a/backend/src/services/profile_preference_service.py b/backend/src/services/profile_preference_service.py index 65e0dc71..5ad2f185 100644 --- a/backend/src/services/profile_preference_service.py +++ b/backend/src/services/profile_preference_service.py @@ -1,4 +1,5 @@ # #region profile_preference_service [C:4] [TYPE Module] [SEMANTICS profile,preference,crud,persistence] +# @defgroup Services Module group. # @BRIEF Profile preference persistence — read, update, and normalize user dashboard filter preferences # with validation, encryption of git tokens, and deterministic default construction. # @LAYER Domain @@ -48,6 +49,7 @@ from .security_badge_service import SecurityBadgeService # #region ProfilePreferenceService [C:4] [TYPE Class] [SEMANTICS preference,crud,validation,encryption] +# @defgroup Services Module group. # @BRIEF Handles profile preference persistence, validation, and token encryption. # @RELATION DEPENDS_ON -> [AuthRepository] # @RELATION DEPENDS_ON -> [EncryptionManager] @@ -71,6 +73,7 @@ class ProfilePreferenceService: # #endregion __init__ # #region get_my_preference [TYPE Function] +# @ingroup Services # @BRIEF Return current user's persisted preference or default non-configured view. # @PRE current_user is authenticated. # @POST Returned payload belongs to current_user only. @@ -100,6 +103,7 @@ class ProfilePreferenceService: # #endregion get_my_preference # #region get_dashboard_filter_binding [TYPE Function] +# @ingroup Services # @BRIEF Return only dashboard-filter fields required by dashboards listing hot path. # @PRE current_user is authenticated. # @POST Returns normalized username and profile-default filter toggles without security summary expansion. @@ -133,6 +137,7 @@ class ProfilePreferenceService: # #endregion get_dashboard_filter_binding # #region update_my_preference [TYPE Function] +# @ingroup Services # @BRIEF Validate and persist current user's profile preference in self-scoped mode. # @PRE current_user is authenticated and payload is provided. # @POST Preference row for current_user is created/updated when validation passes. diff --git a/backend/src/services/profile_service.py b/backend/src/services/profile_service.py index 04885084..a5e765f5 100644 --- a/backend/src/services/profile_service.py +++ b/backend/src/services/profile_service.py @@ -1,4 +1,5 @@ # #region profile_service [C:5] [TYPE Module] [SEMANTICS sqlalchemy, profile, preference, superset, lookup] +# @defgroup Services Module group. # # @BRIEF Composite facade orchestrating profile preference persistence, Superset account lookup, # security badges, and deterministic actor matching by delegating to focused sub-services. @@ -70,6 +71,7 @@ __all__ = [ # #region ProfileService [C:4] [TYPE Class] [SEMANTICS profile,facade,delegate,coordinator] +# @defgroup Services Module group. # @BRIEF Facade that composes ProfilePreferenceService, SupersetLookupService, and # SecurityBadgeService into a single interface for backward compatibility. # @RELATION DEPENDS_ON -> [ProfilePreferenceService] diff --git a/backend/src/services/profile_utils.py b/backend/src/services/profile_utils.py index f8d15e45..16016caf 100644 --- a/backend/src/services/profile_utils.py +++ b/backend/src/services/profile_utils.py @@ -1,4 +1,5 @@ # #region ProfileUtils [C:2] [TYPE Module] [SEMANTICS string,normalization,utility,sanitize] +# @defgroup Services Module group. # @BRIEF Pure utility helpers for profile data sanitization, normalization, and secret masking. # Also contains shared exception classes to avoid circular imports between profile sub-modules. # @LAYER Domain @@ -15,6 +16,7 @@ from typing import Any # #region ProfileValidationError [C:2] [TYPE Class] +# @defgroup Services Module group. # @RELATION INHERITS -> [EXT:Python:Exception] # @BRIEF Domain validation error for profile preference update requests. class ProfileValidationError(Exception): @@ -25,6 +27,7 @@ class ProfileValidationError(Exception): # #region EnvironmentNotFoundError [C:2] [TYPE Class] +# @defgroup Services Module group. # @RELATION INHERITS -> [EXT:Python:Exception] # @BRIEF Raised when environment_id from lookup request is unknown in app configuration. class EnvironmentNotFoundError(Exception): @@ -33,6 +36,7 @@ class EnvironmentNotFoundError(Exception): # #region ProfileAuthorizationError [C:2] [TYPE Class] +# @defgroup Services Module group. # @RELATION INHERITS -> [EXT:Python:Exception] # @BRIEF Raised when caller attempts cross-user preference mutation. class ProfileAuthorizationError(Exception): @@ -119,6 +123,7 @@ def mask_secret_value(secret: str | None) -> str | None: # #region validate_update_payload [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Validate username/toggle constraints for preference mutation. # @RELATION CALLS -> [sanitize_username] # @RELATION CALLS -> [sanitize_text] @@ -176,6 +181,7 @@ def validate_update_payload( # #region build_default_preference [C:2] [TYPE Function] [SEMANTICS preference,default,unconfigured] +# @ingroup Services # @BRIEF Build non-persisted default preference DTO for unconfigured users. # @RELATION DEPENDS_ON -> [ProfilePreference] def build_default_preference(user_id: str) -> Any: @@ -205,6 +211,7 @@ def build_default_preference(user_id: str) -> Any: # #region normalize_owner_tokens [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Normalize owners payload into deduplicated lower-cased tokens. # @RELATION CALLS -> [normalize_username] def normalize_owner_tokens(owners: Iterable[Any] | None) -> list[str]: diff --git a/backend/src/services/rbac_permission_catalog.py b/backend/src/services/rbac_permission_catalog.py index 7948234b..5b3f2ea4 100644 --- a/backend/src/services/rbac_permission_catalog.py +++ b/backend/src/services/rbac_permission_catalog.py @@ -1,4 +1,5 @@ # #region rbac_permission_catalog [C:2] [TYPE Module] [SEMANTICS rbac, permission, discovery, route, sync] +# @defgroup Services Module group. # # @BRIEF Discovers declared RBAC permissions from API routes/plugins and synchronizes them into auth database. @@ -13,6 +14,7 @@ from ..core.logger import belief_scope, logger from ..models.auth import Permission # #region HAS_PERMISSION_PATTERN [TYPE Constant] +# @ingroup Services # @BRIEF Regex pattern for extracting has_permission("resource", "ACTION") declarations. HAS_PERMISSION_PATTERN = re.compile( r"""has_permission\(\s*['"]([^'"]+)['"]\s*,\s*['"]([A-Z]+)['"]\s*\)""" @@ -20,6 +22,7 @@ HAS_PERMISSION_PATTERN = re.compile( # #endregion HAS_PERMISSION_PATTERN # #region ROUTES_DIR [TYPE Constant] +# @ingroup Services # @BRIEF Absolute directory path where API route RBAC declarations are defined. ROUTES_DIR = Path(__file__).resolve().parent.parent / "api" / "routes" # #endregion ROUTES_DIR @@ -122,6 +125,7 @@ def _discover_plugin_execute_permissions_cached( # #region discover_declared_permissions [C:4] [TYPE Function] +# @ingroup Services # @BRIEF Builds canonical RBAC permission catalog from routes and plugin registry. # @PRE plugin_loader may be provided for dynamic task plugin permission discovery. # @POST Returns union of route-declared and dynamic plugin EXECUTE permissions. @@ -143,6 +147,7 @@ def discover_declared_permissions(plugin_loader=None) -> set[tuple[str, str]]: # #region sync_permission_catalog [C:4] [TYPE Function] +# @ingroup Services # @BRIEF Persists missing RBAC permission pairs into auth database. # @PRE db is a valid SQLAlchemy session bound to auth database. # @PRE declared_permissions is an iterable of (resource, action) tuples. diff --git a/backend/src/services/reports/__init__.py b/backend/src/services/reports/__init__.py index 0e156fc0..d3ad273c 100644 --- a/backend/src/services/reports/__init__.py +++ b/backend/src/services/reports/__init__.py @@ -1,3 +1,4 @@ # #region reports [TYPE Package] [SEMANTICS report, package, service, root] +# @ingroup Services # @BRIEF Report service package root. # #endregion reports diff --git a/backend/src/services/reports/normalizer.py b/backend/src/services/reports/normalizer.py index 3ea07e6c..2f4566d7 100644 --- a/backend/src/services/reports/normalizer.py +++ b/backend/src/services/reports/normalizer.py @@ -1,4 +1,5 @@ # #region normalizer [C:5] [TYPE Module] [SEMANTICS pydantic, report, task, normalize, status] +# @defgroup Services Module group. # @BRIEF Convert task manager task objects into canonical unified TaskReport entities with deterministic fallback behavior. # @LAYER Domain # @RELATION DEPENDS_ON -> [EXT:frontend:TaskModel] @@ -20,6 +21,7 @@ from .type_profiles import get_type_profile, resolve_task_type # #region status_to_report_status [TYPE Function] +# @ingroup Services # @BRIEF Normalize internal task status to canonical report status. # @PRE status may be known or unknown string/enum value. # @POST Always returns one of canonical ReportStatus values. @@ -37,6 +39,7 @@ def status_to_report_status(status: Any) -> ReportStatus: # #region build_summary [TYPE Function] +# @ingroup Services # @BRIEF Build deterministic user-facing summary from task payload and status. # @PRE report_status is canonical; plugin_id may be unknown. # @POST Returns non-empty summary text. @@ -59,6 +62,7 @@ def build_summary(task: Task, report_status: ReportStatus) -> str: # #region extract_error_context [TYPE Function] +# @ingroup Services # @BRIEF Extract normalized error context and next actions for failed/partial reports. # @PRE task is a valid Task object. # @POST Returns ErrorContext for failed/partial when context exists; otherwise None. @@ -100,6 +104,7 @@ def extract_error_context(task: Task, report_status: ReportStatus) -> ErrorConte # #region normalize_task_report [TYPE Function] +# @ingroup Services # @BRIEF Convert one Task to canonical TaskReport envelope. # @PRE task has valid id and plugin_id fields. # @POST Returns TaskReport with required fields and deterministic fallback behavior. diff --git a/backend/src/services/reports/report_service.py b/backend/src/services/reports/report_service.py index aac0d7e8..98b46c9d 100644 --- a/backend/src/services/reports/report_service.py +++ b/backend/src/services/reports/report_service.py @@ -1,4 +1,5 @@ # #region report_service [C:5] [TYPE Module] [SEMANTICS report, task, filter, paginate, aggregate] +# @defgroup Services Module group. # @BRIEF Aggregate, normalize, filter, and paginate task reports for unified list/detail API use cases. # @LAYER Domain # @RELATION DEPENDS_ON -> [TaskManager] @@ -31,6 +32,7 @@ from .normalizer import normalize_task_report # #region ReportsService [C:5] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Service layer for list/detail report retrieval and normalization. # @PRE TaskManager dependency is initialized. # @POST Provides deterministic list/detail report responses. diff --git a/backend/src/services/reports/type_profiles.py b/backend/src/services/reports/type_profiles.py index 3201bd27..05e1f580 100644 --- a/backend/src/services/reports/type_profiles.py +++ b/backend/src/services/reports/type_profiles.py @@ -1,4 +1,5 @@ # #region type_profiles [C:2] [TYPE Module] [SEMANTICS report, task-type, mapping, profile, fallback] +# @defgroup Services Module group. # @BRIEF Deterministic mapping of plugin/task identifiers to canonical report task types and fallback profile metadata. from typing import Any @@ -7,6 +8,7 @@ from ...core.logger import belief_scope from ...models.report import TaskType # #region PLUGIN_TO_TASK_TYPE [TYPE Data] +# @ingroup Services # @BRIEF Maps plugin identifiers to normalized report task types. PLUGIN_TO_TASK_TYPE: dict[str, TaskType] = { "llm_dashboard_validation": TaskType.LLM_VERIFICATION, @@ -19,6 +21,7 @@ PLUGIN_TO_TASK_TYPE: dict[str, TaskType] = { # #endregion PLUGIN_TO_TASK_TYPE # #region TASK_TYPE_PROFILES [TYPE Data] +# @ingroup Services # @BRIEF Profile metadata registry for each normalized task type. TASK_TYPE_PROFILES: dict[TaskType, dict[str, Any]] = { TaskType.LLM_VERIFICATION: { @@ -68,6 +71,7 @@ TASK_TYPE_PROFILES: dict[TaskType, dict[str, Any]] = { # #region resolve_task_type [TYPE Function] +# @ingroup Services # @BRIEF Resolve canonical task type from plugin/task identifier with guaranteed fallback. # @PRE plugin_id may be None or unknown. # @POST Always returns one of TaskType enum values. @@ -94,6 +98,7 @@ def resolve_task_type(plugin_id: str | None) -> TaskType: # #region get_type_profile [TYPE Function] +# @ingroup Services # @BRIEF Return deterministic profile metadata for a task type. # @PRE task_type may be known or unknown. # @POST Returns a profile dict and never raises for unknown types. diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index cd3a3205..44cdfefb 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -1,4 +1,5 @@ # #region ResourceServiceModule [C:5] [TYPE Module] [SEMANTICS resource, git, task, status, superset] +# @defgroup Services Module group. # @BRIEF Shared service for fetching resource data with Git status and task status # @LAYER Service # @RELATION DEPENDS_ON -> [SupersetClient] @@ -20,6 +21,7 @@ from ..services.git_service import GitService # #region ResourceService [C:3] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Provides centralized access to resource data with enhanced metadata # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [GitService] diff --git a/backend/src/services/security_badge_service.py b/backend/src/services/security_badge_service.py index 231e46c1..08b2f902 100644 --- a/backend/src/services/security_badge_service.py +++ b/backend/src/services/security_badge_service.py @@ -1,4 +1,5 @@ # #region security_badge_service [C:4] [TYPE Module] [SEMANTICS profile,security,permissions,badges] +# @defgroup Services Module group. # @BRIEF Builds security summary and permission badges for profile UI — role extraction, # permission pair collection, and catalog formatting. # @LAYER Domain @@ -25,6 +26,7 @@ from .rbac_permission_catalog import discover_declared_permissions # #region SecurityBadgeService [C:4] [TYPE Class] [SEMANTICS security,permission,badge,profile] +# @defgroup Services Module group. # @BRIEF Builds security summary with role names and permission badges for profile UI display. # @RELATION DEPENDS_ON -> [User] # @RELATION CALLS -> [discover_declared_permissions] @@ -40,6 +42,7 @@ class SecurityBadgeService: # #endregion __init__ # #region build_security_summary [TYPE Function] +# @ingroup Services # @BRIEF Build read-only security snapshot with role and permission badges. # @PRE current_user is authenticated. # @POST Returns deterministic security projection for profile UI. diff --git a/backend/src/services/sql_table_extractor.py b/backend/src/services/sql_table_extractor.py index 629861b6..5bb766c0 100644 --- a/backend/src/services/sql_table_extractor.py +++ b/backend/src/services/sql_table_extractor.py @@ -1,4 +1,5 @@ # #region SqlTableExtractorModule [C:2] [TYPE Module] [SEMANTICS sql, jinja, table, extraction, parsing] +# @defgroup Services Module group. # @BRIEF Three-phase SQL+Jinja table name extractor for virtual datasets as per FR-003. # Phase 1: Detect Jinja spans vs SQL spans # Phase 2: In Jinja spans, extract "schema.table" from string values @@ -41,6 +42,7 @@ _SCHEMA_TABLE_RE = re.compile( # #region detect_jinja_spans [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Phase 1: Split raw SQL text into Jinja spans and SQL spans. # @PRE raw_sql is a string (possibly empty). # @POST Returns a list of (span_type: str, text: str) tuples. @@ -94,6 +96,7 @@ def detect_jinja_spans(raw_sql: str) -> list[tuple[str, str]]: # #region extract_tables_from_jinja [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Phase 2: Extract "schema.table" references from Jinja string values. # Looks for patterns like "raw.sales" inside Jinja text. # @PRE jinja_text is a string from a Jinja span. @@ -126,6 +129,7 @@ def is_string_literal(token: Token) -> bool: # #region extract_tables_from_sql_span [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Phase 3: Extract schema.table references from SQL text using regex + sqlparse filtering. # Uses regex to find all schema.table candidates, then sqlparse to filter out # false positives inside string literals. @@ -188,6 +192,7 @@ def extract_tables_from_sql_span(sql_text: str) -> set[str]: # #region extract_tables_from_sql [C:2] [TYPE Function] +# @ingroup Services # @BRIEF Three-phase entry point: extract all schema.table references from raw SQL+Jinja text. # @PRE raw_sql is a string (possibly empty). # @POST Returns a set of lowercased fully-qualified table names (schema.table format). diff --git a/backend/src/services/superset_lookup_service.py b/backend/src/services/superset_lookup_service.py index 38519192..100e946c 100644 --- a/backend/src/services/superset_lookup_service.py +++ b/backend/src/services/superset_lookup_service.py @@ -1,4 +1,5 @@ # #region superset_lookup_service [C:4] [TYPE Module] [SEMANTICS superset,account,lookup,environment] +# @defgroup Services Module group. # @BRIEF Environment-scoped Superset account lookup with degradation fallback for network failures. # @LAYER Domain # @RELATION DEPENDS_ON -> [SupersetClient] @@ -29,6 +30,7 @@ from .profile_utils import EnvironmentNotFoundError # #region SupersetLookupService [C:4] [TYPE Class] [SEMANTICS superset,lookup,environment,degradation] +# @defgroup Services Module group. # @BRIEF Resolves environments and queries Superset users in selected environment, # returning canonical account candidates with degradation fallback. # @RELATION DEPENDS_ON -> [SupersetClient] @@ -46,6 +48,7 @@ class SupersetLookupService: # #endregion __init__ # #region lookup_superset_accounts [TYPE Function] +# @ingroup Services # @BRIEF Query Superset users in selected environment and project canonical account candidates. # @SIDE_EFFECT Асинхронный HTTP-вызов к Superset API. # @PRE current_user is authenticated and environment_id exists. diff --git a/backend/src/services/validation_service.py b/backend/src/services/validation_service.py index ecad950e..7f63b7ba 100644 --- a/backend/src/services/validation_service.py +++ b/backend/src/services/validation_service.py @@ -1,4 +1,5 @@ # #region ValidationService [C:4] [TYPE Module] [SEMANTICS validation, service, sqlalchemy, task] +# @defgroup Services Module group. # @BRIEF Business logic for validation task CRUD and trigger-run — v2 with source-aware policy management. # @LAYER Service # @RELATION DEPENDS_ON -> [LLMProviderService] @@ -62,6 +63,7 @@ def _format_time(value: time | None) -> str | None: # #region ValidationTaskService [C:4] [TYPE Class] +# @defgroup Services Module group. # @BRIEF Validation task (policy) CRUD with provider validation and trigger-run. # @RATIONALE Service-layer class for validation task CRUD with provider validation and trigger-run. # Separates API layer concerns from business logic; enables unit testing without HTTP. @@ -376,6 +378,7 @@ class ValidationTaskService: # #endregion _record_to_dict # #region list_tasks [C:3] [TYPE Function] +# @ingroup Services # @RATIONALE Uses query-time filtering with optional is_active/environment_id params rather # than post-filtering results in memory, reducing overhead with large task counts. # @REJECTED In-memory filtering after loading all rows rejected — would be inefficient with @@ -408,6 +411,7 @@ class ValidationTaskService: # #endregion list_tasks # #region create_task [C:4] [TYPE Function] +# @ingroup Services # @RATIONALE Validates provider (multimodal check depends on screenshot_enabled flag) and # environment before persisting the policy. Resolves sources from explicit input # or falls back to dashboard_ids for backward compatibility. @@ -465,6 +469,7 @@ class ValidationTaskService: # #endregion create_task # #region get_task [C:3] [TYPE Function] +# @ingroup Services # @RATIONALE Returns full task response including last run status via ValidationRun — gives # the UI immediate visibility into task health without a second API call. # @REJECTED Separate query for last run status rejected — double-query pattern would add @@ -510,6 +515,7 @@ class ValidationTaskService: # #endregion _apply_sources_update # #region update_task [C:3] [TYPE Function] +# @ingroup Services # @RATIONALE Re-validates provider/environment only when those fields change (via exclude_unset), # avoiding unnecessary DB calls on partial updates. Delegates time parsing and source # replacement to extracted methods to stay within CC ≤ 10. @@ -544,6 +550,7 @@ class ValidationTaskService: # #endregion update_task # #region delete_task [C:3] [TYPE Function] +# @ingroup Services # @RATIONALE Sources are cascade-deleted via SQLAlchemy relationship (cascade="all, delete-orphan"). # When delete_runs=True, also removes ValidationRun and their ValidationRecord children. # @REJECTED Always-cascade delete rejected — would destroy run history unexpectedly. Explicit @@ -582,6 +589,7 @@ class ValidationTaskService: # #endregion delete_task # #region trigger_run [C:4] [TYPE Function] +# @ingroup Services # @RATIONALE Creates a ValidationRun record as an aggregate execution container. Checks FR-054 # (no concurrent running runs for the same policy) and global_validation_worker_limit # before spawning. Passes all resolved dashboard sources to the task manager plugin. @@ -690,6 +698,7 @@ class ValidationTaskService: # #endregion trigger_run # #region get_run_history [C:3] [TYPE Function] +# @ingroup Services # @BRIEF Return paginated ValidationRun list for a policy, ordered by started_at desc. # @RATIONALE Uses ValidationRun (aggregate per-run) rather than individual ValidationRecords, # giving a correct v2 view of execution history with aggregate pass/fail counts. @@ -709,6 +718,7 @@ class ValidationTaskService: # #endregion get_run_history # #region list_all_runs [C:2] [TYPE Function] +# @ingroup Services # @BRIEF List runs across ALL tasks with optional filters (replaces v1 cross-task endpoint). def list_all_runs( self, @@ -739,6 +749,7 @@ class ValidationTaskService: # #endregion list_all_runs # #region get_run_detail [C:3] [TYPE Function] +# @ingroup Services # @BRIEF Return full ValidationRun with all its ValidationRecords, including v2 metadata # (token_usage, timings, execution_path, dataset_health, etc.). # @RATIONALE Loads records with eager filtering on run_id for a complete picture of a single diff --git a/frontend/src/lib/Counter.svelte b/frontend/src/lib/Counter.svelte index e68869a2..80f0895d 100755 --- a/frontend/src/lib/Counter.svelte +++ b/frontend/src/lib/Counter.svelte @@ -1,4 +1,5 @@ + + diff --git a/frontend/src/lib/components/MaintenanceEventsTable.svelte b/frontend/src/lib/components/MaintenanceEventsTable.svelte index f02f5384..60d3a6fa 100644 --- a/frontend/src/lib/components/MaintenanceEventsTable.svelte +++ b/frontend/src/lib/components/MaintenanceEventsTable.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/lib/components/MaintenanceSettingsPanel.svelte b/frontend/src/lib/components/MaintenanceSettingsPanel.svelte index 51ec039d..f855fa30 100644 --- a/frontend/src/lib/components/MaintenanceSettingsPanel.svelte +++ b/frontend/src/lib/components/MaintenanceSettingsPanel.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/lib/components/StartMaintenanceForm.svelte b/frontend/src/lib/components/StartMaintenanceForm.svelte index 65823fe0..bb5f6330 100644 --- a/frontend/src/lib/components/StartMaintenanceForm.svelte +++ b/frontend/src/lib/components/StartMaintenanceForm.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/lib/components/assistant/AssistantChatPanel.svelte b/frontend/src/lib/components/assistant/AssistantChatPanel.svelte index 208927ef..d19ed867 100644 --- a/frontend/src/lib/components/assistant/AssistantChatPanel.svelte +++ b/frontend/src/lib/components/assistant/AssistantChatPanel.svelte @@ -1,4 +1,5 @@ + + diff --git a/frontend/src/routes/profile/+page.svelte b/frontend/src/routes/profile/+page.svelte index 0e3c657c..65acd797 100644 --- a/frontend/src/routes/profile/+page.svelte +++ b/frontend/src/routes/profile/+page.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/routes/reports/+page.svelte b/frontend/src/routes/reports/+page.svelte index e11c9ffc..ff1a47e3 100644 --- a/frontend/src/routes/reports/+page.svelte +++ b/frontend/src/routes/reports/+page.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/routes/reports/llm/[taskId]/+page.svelte b/frontend/src/routes/reports/llm/[taskId]/+page.svelte index 13d44070..9c218a30 100644 --- a/frontend/src/routes/reports/llm/[taskId]/+page.svelte +++ b/frontend/src/routes/reports/llm/[taskId]/+page.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/routes/settings/+page.svelte b/frontend/src/routes/settings/+page.svelte index e64c9ea5..0d49f6e7 100644 --- a/frontend/src/routes/settings/+page.svelte +++ b/frontend/src/routes/settings/+page.svelte @@ -1,4 +1,5 @@ + diff --git a/frontend/src/routes/settings/EnvironmentsTab.svelte b/frontend/src/routes/settings/EnvironmentsTab.svelte index 33856e08..9e8c8a58 100644 --- a/frontend/src/routes/settings/EnvironmentsTab.svelte +++ b/frontend/src/routes/settings/EnvironmentsTab.svelte @@ -1,4 +1,5 @@ +