diff --git a/.axiom/axiom_config.yaml b/.axiom/axiom_config.yaml index 7e14ac89..ebfc9dc4 100644 --- a/.axiom/axiom_config.yaml +++ b/.axiom/axiom_config.yaml @@ -1,8 +1,9 @@ # #region AnchorConfig [C:3] [TYPE Block] [SEMANTICS config,anchor] # @BRIEF Якорный синтаксис — глобальный формат и переопределения по директориям. -# @RELATION BINDS_TO -> [Std.Semantics.Core §II] -# @RATIONALE По умолчанию #region/#endregion — новый код пишется в region-формате. -# Legacy DEF и Doc-формат brace признаются для обратной совместимости. +# @RELATION BINDS_TO -> [Std.Semantics.Core] +# +# По умолчанию #region/#endregion — новый код пишется в region-формате. +# Legacy DEF и Doc-формат brace признаются для обратной совместимости. anchor: format: region overrides: @@ -11,7 +12,7 @@ anchor: syntax: {} # #endregion AnchorConfig -# #region IndexingConfig [C:3] [TYPE Block] [SEMANTICS config,indexing] +# #region IndexingConfig [C:2] [TYPE Block] [SEMANTICS config,indexing] # @BRIEF Правила обхода файлов — include/exclude паттерны и source/doc директории. indexing: include: [] @@ -39,7 +40,9 @@ indexing: # #region ComplexityRules [C:5] [TYPE Block] [SEMANTICS config,rules,validation] # @BRIEF Обязательные и запрещённые тэги по уровням сложности C1-C5. -# @RELATION BINDS_TO -> [Std.Semantics.Core §III] +# @RELATION BINDS_TO -> [Std.Semantics.Core] +# @RELATION DEPENDS_ON -> [TagSchema] +# @INVARIANT Каждый тэг в required/forbidden списках обязан иметь определение в TagSchema. # @RATIONALE Каждый уровень строго контролирует допустимый набор тэгов — это предотвращает verbosity и структурную эрозию (INV_7). # LAYER и SEMANTICS — Module-специфичные тэги; не входят в базовую шкалу сложности core. # BRIEF указан как alias PURPOSE — явно в forbidden для C1, т.к. C1 запрещает все content-тэги. @@ -111,7 +114,10 @@ complexity_rules: # #region ContractTypeOverrides [C:3] [TYPE Block] [SEMANTICS config,adr,override] # @BRIEF Особые правила валидации для ADR, Component и Tombstone. -# @RATIONALE ADR не имеет COMPLEXITY (это всегда архитектурное решение). Component имеет @UX_STATE вместо @PRE/@POST на C3. Tombstone — минимальный контракт-заглушка. +# @RELATION DEPENDS_ON -> [ComplexityRules] +# @RELATION DEPENDS_ON -> [TagSchema] +# +# ADR не имеет COMPLEXITY (это всегда архитектурное решение). Component имеет @UX_STATE вместо @PRE/@POST на C3. Tombstone — минимальный контракт-заглушка. contract_type_overrides: ADR: required: @@ -180,10 +186,49 @@ contract_type_overrides: - RELATION - LAYER - SEMANTICS + Block: + '3': + required: + - PURPOSE + - RELATION + forbidden: + - PRE + - POST + - SIDE_EFFECT + - DATA_CONTRACT + - INVARIANT + - RATIONALE + - REJECTED + '4': + required: + - PURPOSE + - RELATION + - RATIONALE + - REJECTED + forbidden: + - PRE + - POST + - SIDE_EFFECT + - DATA_CONTRACT + - INVARIANT + '5': + required: + - PURPOSE + - RELATION + - RATIONALE + - REJECTED + - INVARIANT + forbidden: + - PRE + - POST + - SIDE_EFFECT + - DATA_CONTRACT # #endregion ContractTypeOverrides # #region TagSchema [C:5] [TYPE Block] [SEMANTICS config,tags,schema] # @BRIEF Полная схема GRACE-тэгов: типы, алиасы, enum-ы, предикаты, применимость. +# @RELATION DEPENDS_ON -> [ComplexityRules] +# @INVARIANT Каждый тэг, referenced в complexity_rules и contract_type_overrides, имеет определение в tags. # @RATIONALE Централизованная схема тэгов — единый источник истины для парсера, валидатора и MCP-тулов. Расширяется через alias_for без изменения канонических имён. # @RATIONALE LAYER.enum +Service: semantics-python использует @LAYER Service в canonical C3 Module примере (dashboard_migration). # @RATIONALE SEMANTICS/PRE/POST/SIDE_EFFECT/DATA_CONTRACT/INVARIANT +Agent: Core §II декларирует Agent как валидный тип контракта; Fullstack.Coder и другие Agent-контракты используют эти тэги. @@ -271,19 +316,20 @@ tags: SEMANTICS: type: array multiline: false - description: Семантические маркеры (домен, зона ответственности). Ортогональный — допустим на любом C-level для Module. + description: Семантические маркеры (домен, зона ответственности). Ортогональный — допустим на любом C-level для Module, Block, Skill, Agent. separator: ',' is_reference: false enum: [] allowed_predicates: [] contract_types: - Module + - Block - Skill - Agent protected: false orthogonal: true decision_memory: false - alias_for: null + alias_for: null SIDE_EFFECT: type: string multiline: false @@ -448,6 +494,167 @@ tags: orthogonal: false decision_memory: false alias_for: null + RELATION: + type: string + multiline: false + description: 'Графовая зависимость: PREDICATE -> [TargetId]. Predicates: DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES. Обязателен с C3.' + separator: null + is_reference: true + enum: [] + allowed_predicates: + - DEPENDS_ON + - CALLS + - INHERITS + - IMPLEMENTS + - DISPATCHES + - BINDS_TO + - CALLED_BY + - VERIFIES + contract_types: + - Module + - Function + - Class + - Component + - Block + - ADR + - Skill + - Agent + protected: false + orthogonal: false + decision_memory: false + alias_for: null + PRE: + type: string + multiline: true + description: 'Предусловия выполнения. Должны быть проверены в коде через if/raise (не assert). Обязателен с C4.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Function + - Class + - Component + - Agent + protected: false + orthogonal: false + decision_memory: false + alias_for: null + POST: + type: string + multiline: true + description: 'Гарантии результата. Обязателен с C4. Не может быть ослаблен без проверки upstream @RELATION зависимостей.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Function + - Class + - Component + - Agent + protected: false + orthogonal: false + decision_memory: false + alias_for: null + RATIONALE: + type: string + multiline: true + description: 'Обоснование выбранного решения. C5-only. Часть Decision Memory — расширяет логический скелет архитектуры.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Function + - Class + - Component + - Block + - ADR + - Skill + - Agent + protected: false + orthogonal: false + decision_memory: true + alias_for: null + REJECTED: + type: string + multiline: true + description: 'Отвергнутый альтернативный путь с указанием причины. C5-only. Часть Decision Memory — запрещает воскрешение rejected-пути.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Function + - Class + - Component + - Block + - ADR + - Skill + - Agent + protected: false + orthogonal: false + decision_memory: true + alias_for: null + DATA_CONTRACT: + type: string + multiline: false + description: 'DTO-маппинг: Input -> InputDTO, Output -> OutputDTO. Обязателен с C5.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Function + - Class + - Component + - Agent + protected: false + orthogonal: false + decision_memory: false + alias_for: null + INVARIANT: + type: string + multiline: true + description: 'Инвариант — условие, которое всегда истинно для данного контракта. Обязателен с C5.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Function + - Class + - Component + - Block + - Skill + - Agent + protected: false + orthogonal: false + decision_memory: false + alias_for: null + LAYER: + type: string + multiline: false + description: 'Архитектурный слой: Service, UI, Infrastructure, Domain. Ортогональный — Module/Skill/Agent-специфичный.' + separator: null + is_reference: false + enum: [] + allowed_predicates: [] + contract_types: + - Module + - Skill + - Agent + protected: false + orthogonal: true + decision_memory: false + alias_for: null # #endregion TagSchema # #region InfrastructureConfig [C:2] [TYPE Block] [SEMANTICS config,embedding,http] diff --git a/.axiom/runtime/belief_events.jsonl b/.axiom/runtime/belief_events.jsonl index 3cf8d367..1f4e0779 100644 --- a/.axiom/runtime/belief_events.jsonl +++ b/.axiom/runtime/belief_events.jsonl @@ -10,3 +10,17 @@ {"timestamp":1778586506.114,"event_type":"belief_reason","component":"Axiom:Services:Contract:Rebuild:SemanticIndex","data":{"depth":1,"extra":{"rebuild_mode":"full","refresh_embeddings":true},"message":"Rebuilding the semantic index snapshot for the workspace."}} {"recorded_at":"2026-05-12T11:49:05.732407581Z","anchor_id":"Axiom:Services:Contract:Rebuild:SemanticIndex","marker":"reflect","message":"Semantic index snapshot persisted.","depth":1,"extra":{"contract_count":3029,"index_path":"/home/busya/dev/ss-tools/.axiom/semantic_index/index.json"}} {"timestamp":1778586545.732,"event_type":"belief_reflect","component":"Axiom:Services:Contract:Rebuild:SemanticIndex","data":{"depth":1,"extra":{"contract_count":3029,"index_path":"/home/busya/dev/ss-tools/.axiom/semantic_index/index.json"},"message":"Semantic index snapshot persisted."}} +{"recorded_at":"2026-05-12T14:10:49.026642566Z","anchor_id":"Axiom:Services:Artifact:PatchWorkspaceFile","marker":"reason","message":"Applying a workspace file patch after producing a preview diff.","depth":1,"extra":{"path":"backend/src/core/logger.py","target_mode":"exact_text"}} +{"timestamp":1778595049.026,"event_type":"belief_reason","component":"Axiom:Services:Artifact:PatchWorkspaceFile","data":{"depth":1,"extra":{"path":"backend/src/core/logger.py","target_mode":"exact_text"},"message":"Applying a workspace file patch after producing a preview diff."}} +{"recorded_at":"2026-05-12T14:10:49.026784231Z","anchor_id":"Axiom:Services:Checkpoint:CreateCheckpoint","marker":"reason","message":"Capturing rollback state before a workspace mutation.","depth":2,"extra":{"path_count":1,"reason":"patch backend/src/core/logger.py"}} +{"timestamp":1778595049.026,"event_type":"belief_reason","component":"Axiom:Services:Checkpoint:CreateCheckpoint","data":{"depth":2,"extra":{"path_count":1,"reason":"patch backend/src/core/logger.py"},"message":"Capturing rollback state before a workspace mutation."}} +{"recorded_at":"2026-05-12T14:10:49.028640082Z","anchor_id":"Axiom:Services:Checkpoint:CreateCheckpoint","marker":"reflect","message":"Checkpoint manifest persisted for future rollback.","depth":2,"extra":{"affected_paths":1,"checkpoint_id":"68541792-8853-40af-b26b-f15dd43fbbca"}} +{"timestamp":1778595049.028,"event_type":"belief_reflect","component":"Axiom:Services:Checkpoint:CreateCheckpoint","data":{"depth":2,"extra":{"affected_paths":1,"checkpoint_id":"68541792-8853-40af-b26b-f15dd43fbbca"},"message":"Checkpoint manifest persisted for future rollback."}} +{"recorded_at":"2026-05-12T14:10:49.052987590Z","anchor_id":"Axiom:Services:Artifact:PatchWorkspaceFile","marker":"reflect","message":"Workspace file patch committed and checkpointed.","depth":1,"extra":{"checkpoint_id":"68541792-8853-40af-b26b-f15dd43fbbca","path":"backend/src/core/logger.py"}} +{"timestamp":1778595049.053,"event_type":"belief_reflect","component":"Axiom:Services:Artifact:PatchWorkspaceFile","data":{"depth":1,"extra":{"checkpoint_id":"68541792-8853-40af-b26b-f15dd43fbbca","path":"backend/src/core/logger.py"},"message":"Workspace file patch committed and checkpointed."}} +{"recorded_at":"2026-05-12T14:12:59.640254945Z","anchor_id":"Axiom:Services:Contract:Rebuild:SemanticIndex","marker":"reason","message":"Rebuilding the semantic index snapshot for the workspace.","depth":1,"extra":{"rebuild_mode":"full","refresh_embeddings":true}} +{"timestamp":1778595179.64,"event_type":"belief_reason","component":"Axiom:Services:Contract:Rebuild:SemanticIndex","data":{"depth":1,"extra":{"rebuild_mode":"full","refresh_embeddings":true},"message":"Rebuilding the semantic index snapshot for the workspace."}} +{"recorded_at":"2026-05-12T14:13:47.436444453Z","anchor_id":"Axiom:Services:Contract:Rebuild:SemanticIndex","marker":"explore","message":"Embedding refresh skipped because no provider is configured.","depth":1,"extra":{"attempted_count":21885}} +{"timestamp":1778595227.436,"event_type":"belief_explore","component":"Axiom:Services:Contract:Rebuild:SemanticIndex","data":{"depth":1,"extra":{"attempted_count":21885},"message":"Embedding refresh skipped because no provider is configured."}} +{"recorded_at":"2026-05-12T14:13:47.492602315Z","anchor_id":"Axiom:Services:Contract:Rebuild:SemanticIndex","marker":"reflect","message":"Semantic index snapshot persisted.","depth":1,"extra":{"contract_count":3052,"index_path":"/home/busya/dev/ss-tools/.axiom/semantic_index/index.json"}} +{"timestamp":1778595227.492,"event_type":"belief_reflect","component":"Axiom:Services:Contract:Rebuild:SemanticIndex","data":{"depth":1,"extra":{"contract_count":3052,"index_path":"/home/busya/dev/ss-tools/.axiom/semantic_index/index.json"},"message":"Semantic index snapshot persisted."}} diff --git a/.axiom/semantic_index/graph.duckdb b/.axiom/semantic_index/graph.duckdb index 484f5f58..f7b78731 100644 Binary files a/.axiom/semantic_index/graph.duckdb and b/.axiom/semantic_index/graph.duckdb differ diff --git a/.axiom/semantic_index/index.json b/.axiom/semantic_index/index.json index 07157488..8c850a90 100644 --- a/.axiom/semantic_index/index.json +++ b/.axiom/semantic_index/index.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-05-12T11:49:05.703662369Z", + "generated_at": "2026-05-12T14:13:36.324499082Z", "root_path": "/home/busya/dev/ss-tools", - "contract_count": 3029, - "edge_count": 2528, - "file_count": 579, + "contract_count": 3052, + "edge_count": 2543, + "file_count": 583, "contracts": [ { "contract_id": "DeleteRunningTasksUtil", @@ -8037,7 +8037,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/admin.py", "start_line": 1, - "end_line": 403, + "end_line": 395, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8150,14 +8150,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:AdminApi:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: api, admin, users, roles, permissions\n# @PURPOSE: Admin API endpoints for user and role management.\n# @LAYER: API\n# @RELATION: [DEPENDS_ON] ->[AuthRepository:Class]\n# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function]\n# @RELATION: [DEPENDS_ON] ->[has_permission:Function]\n#\n# @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope.\n\n# [SECTION: IMPORTS]\nfrom typing import List\nfrom fastapi import APIRouter, Depends, HTTPException, status\nfrom sqlalchemy.orm import Session\nfrom ...core.database import get_auth_db\nfrom ...core.auth.repository import AuthRepository\nfrom ...core.auth.security import get_password_hash\nfrom ...schemas.auth import (\n User as UserSchema,\n UserCreate,\n UserUpdate,\n RoleSchema,\n RoleCreate,\n RoleUpdate,\n PermissionSchema,\n ADGroupMappingSchema,\n ADGroupMappingCreate,\n)\nfrom ...models.auth import User, Role, ADGroupMapping\nfrom ...dependencies import has_permission, get_plugin_loader\nfrom ...core.logger import logger, belief_scope\nfrom ...services.rbac_permission_catalog import (\n discover_declared_permissions,\n sync_permission_catalog,\n)\n# [/SECTION]\n\n# [DEF:router:Variable]\n# @RELATION: DEPENDS_ON -> fastapi.APIRouter\n# @PURPOSE: APIRouter instance for admin routes.\nrouter = APIRouter(prefix=\"/api/admin\", tags=[\"admin\"])\n# [/DEF:router:Variable]\n\n\n# [DEF:list_users:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all registered users.\n# @PRE: Current user has 'Admin' role.\n# @POST: Returns a list of UserSchema objects.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: List[UserSchema] - List of users.\n# @RELATION: CALLS -> User\n@router.get(\"/users\", response_model=List[UserSchema])\nasync def list_users(\n db: Session = Depends(get_auth_db), _=Depends(has_permission(\"admin:users\", \"READ\"))\n):\n with belief_scope(\"api.admin.list_users\"):\n users = db.query(User).all()\n return users\n\n\n# [/DEF:list_users:Function]\n\n\n# [DEF:create_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Creates a new local user.\n# @PRE: Current user has 'Admin' role.\n# @POST: New user is created in the database.\n# @PARAM: user_in (UserCreate) - New user data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: UserSchema - The created user.\n# @RELATION: [CALLS] ->[AuthRepository:Class]\n@router.post(\"/users\", response_model=UserSchema, status_code=status.HTTP_201_CREATED)\nasync def create_user(\n user_in: UserCreate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.create_user\"):\n repo = AuthRepository(db)\n if repo.get_user_by_username(user_in.username):\n raise HTTPException(status_code=400, detail=\"Username already exists\")\n\n new_user = User(\n username=user_in.username,\n email=user_in.email,\n password_hash=get_password_hash(user_in.password),\n auth_source=\"LOCAL\",\n is_active=user_in.is_active,\n )\n\n for role_name in user_in.roles:\n role = repo.get_role_by_name(role_name)\n if role:\n new_user.roles.append(role)\n\n db.add(new_user)\n db.commit()\n db.refresh(new_user)\n return new_user\n\n\n# [/DEF:create_user:Function]\n\n\n# [DEF:update_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Updates an existing user.\n# @PRE: Current user has 'Admin' role.\n# @POST: User record is updated in the database.\n# @PARAM: user_id (str) - Target user UUID.\n# @PARAM: user_in (UserUpdate) - Updated user data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: UserSchema - The updated user profile.\n# @RELATION: CALLS -> AuthRepository\n@router.put(\"/users/{user_id}\", response_model=UserSchema)\nasync def update_user(\n user_id: str,\n user_in: UserUpdate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.update_user\"):\n repo = AuthRepository(db)\n user = repo.get_user_by_id(user_id)\n if not user:\n raise HTTPException(status_code=404, detail=\"User not found\")\n\n if user_in.email is not None:\n user.email = user_in.email\n if user_in.is_active is not None:\n user.is_active = user_in.is_active\n if user_in.password is not None:\n user.password_hash = get_password_hash(user_in.password)\n\n if user_in.roles is not None:\n user.roles = []\n for role_name in user_in.roles:\n role = repo.get_role_by_name(role_name)\n if role:\n user.roles.append(role)\n\n db.commit()\n db.refresh(user)\n return user\n\n\n# [/DEF:update_user:Function]\n\n\n# [DEF:delete_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Deletes a user.\n# @PRE: Current user has 'Admin' role.\n# @POST: User record is removed from the database.\n# @PARAM: user_id (str) - Target user UUID.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: None\n# @RELATION: CALLS -> AuthRepository\n@router.delete(\"/users/{user_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_user(\n user_id: str,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.delete_user\"):\n logger.info(\n f\"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}\"\n )\n repo = AuthRepository(db)\n user = repo.get_user_by_id(user_id)\n if not user:\n logger.warning(\n f\"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}\"\n )\n raise HTTPException(status_code=404, detail=\"User not found\")\n\n logger.info(\n f\"[DEBUG] Found user to delete context={{'username': '{user.username}'}}\"\n )\n db.delete(user)\n db.commit()\n logger.info(\n f\"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}\"\n )\n return None\n\n\n# [/DEF:delete_user:Function]\n\n\n# [DEF:list_roles:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all available roles.\n# @RETURN: List[RoleSchema] - List of roles.\n# @RELATION: [CALLS] ->[Role:Class]\n@router.get(\"/roles\", response_model=List[RoleSchema])\nasync def list_roles(\n db: Session = Depends(get_auth_db), _=Depends(has_permission(\"admin:roles\", \"READ\"))\n):\n with belief_scope(\"api.admin.list_roles\"):\n return db.query(Role).all()\n\n\n# [/DEF:list_roles:Function]\n\n\n# [DEF:create_role:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Creates a new system role with associated permissions.\n# @PRE: Role name must be unique.\n# @POST: New Role record is created in auth.db.\n# @PARAM: role_in (RoleCreate) - New role data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: RoleSchema - The created role.\n# @SIDE_EFFECT: Commits new role and associations to auth.db.\n# @RELATION: [CALLS] ->[get_permission_by_id:Function]\n@router.post(\"/roles\", response_model=RoleSchema, status_code=status.HTTP_201_CREATED)\nasync def create_role(\n role_in: RoleCreate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:roles\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.create_role\"):\n if db.query(Role).filter(Role.name == role_in.name).first():\n raise HTTPException(status_code=400, detail=\"Role already exists\")\n\n new_role = Role(name=role_in.name, description=role_in.description)\n repo = AuthRepository(db)\n\n for perm_id_or_str in role_in.permissions:\n perm = repo.get_permission_by_id(perm_id_or_str)\n if not perm and \":\" in perm_id_or_str:\n res, act = perm_id_or_str.split(\":\", 1)\n perm = repo.get_permission_by_resource_action(res, act)\n\n if perm:\n new_role.permissions.append(perm)\n\n db.add(new_role)\n db.commit()\n db.refresh(new_role)\n return new_role\n\n\n# [/DEF:create_role:Function]\n\n\n# [DEF:update_role:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Updates an existing role's metadata and permissions.\n# @PRE: role_id must be a valid existing role UUID.\n# @POST: Role record is updated in auth.db.\n# @PARAM: role_id (str) - Target role identifier.\n# @PARAM: role_in (RoleUpdate) - Updated role data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: RoleSchema - The updated role.\n# @SIDE_EFFECT: Commits updates to auth.db.\n# @RELATION: [CALLS] ->[get_role_by_id:Function]\n@router.put(\"/roles/{role_id}\", response_model=RoleSchema)\nasync def update_role(\n role_id: str,\n role_in: RoleUpdate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:roles\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.update_role\"):\n repo = AuthRepository(db)\n role = repo.get_role_by_id(role_id)\n if not role:\n raise HTTPException(status_code=404, detail=\"Role not found\")\n\n if role_in.name is not None:\n role.name = role_in.name\n if role_in.description is not None:\n role.description = role_in.description\n\n if role_in.permissions is not None:\n role.permissions = []\n for perm_id_or_str in role_in.permissions:\n perm = repo.get_permission_by_id(perm_id_or_str)\n if not perm and \":\" in perm_id_or_str:\n res, act = perm_id_or_str.split(\":\", 1)\n perm = repo.get_permission_by_resource_action(res, act)\n\n if perm:\n role.permissions.append(perm)\n\n db.commit()\n db.refresh(role)\n return role\n\n\n# [/DEF:update_role:Function]\n\n\n# [DEF:delete_role:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Removes a role from the system.\n# @PRE: role_id must be a valid existing role UUID.\n# @POST: Role record is removed from auth.db.\n# @PARAM: role_id (str) - Target role identifier.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: None\n# @SIDE_EFFECT: Deletes record from auth.db and commits.\n# @RELATION: [CALLS] ->[get_role_by_id:Function]\n@router.delete(\"/roles/{role_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_role(\n role_id: str,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:roles\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.delete_role\"):\n repo = AuthRepository(db)\n role = repo.get_role_by_id(role_id)\n if not role:\n raise HTTPException(status_code=404, detail=\"Role not found\")\n\n db.delete(role)\n db.commit()\n return None\n\n\n# [/DEF:delete_role:Function]\n\n\n# [DEF:list_permissions:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all available system permissions for assignment.\n# @POST: Returns a list of all PermissionSchema objects.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: List[PermissionSchema] - List of permissions.\n# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.list_permissions\n@router.get(\"/permissions\", response_model=List[PermissionSchema])\nasync def list_permissions(\n db: Session = Depends(get_auth_db),\n plugin_loader=Depends(get_plugin_loader),\n _=Depends(has_permission(\"admin:roles\", \"READ\")),\n):\n with belief_scope(\"api.admin.list_permissions\"):\n declared_permissions = discover_declared_permissions(\n plugin_loader=plugin_loader\n )\n inserted_count = sync_permission_catalog(\n db=db, declared_permissions=declared_permissions\n )\n if inserted_count > 0:\n logger.info(\n \"[api.admin.list_permissions][Action] Synchronized %s missing RBAC permissions into auth catalog\",\n inserted_count,\n )\n\n repo = AuthRepository(db)\n return repo.list_permissions()\n\n\n# [/DEF:list_permissions:Function]\n\n\n# [DEF:list_ad_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all AD Group to Role mappings.\n# @RELATION: CALLS -> ADGroupMapping\n@router.get(\"/ad-mappings\", response_model=List[ADGroupMappingSchema])\nasync def list_ad_mappings(\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"api.admin.list_ad_mappings\"):\n return db.query(ADGroupMapping).all()\n\n\n# [/DEF:list_ad_mappings:Function]\n\n\n# [DEF:create_ad_mapping:Function]\n# @RELATION: [DEPENDS_ON] ->[ADGroupMapping:Class]\n# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function]\n# @RELATION: [DEPENDS_ON] ->[has_permission:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Creates a new AD Group mapping.\n@router.post(\"/ad-mappings\", response_model=ADGroupMappingSchema)\nasync def create_ad_mapping(\n mapping_in: ADGroupMappingCreate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.create_ad_mapping\"):\n new_mapping = ADGroupMapping(\n ad_group=mapping_in.ad_group, role_id=mapping_in.role_id\n )\n db.add(new_mapping)\n db.commit()\n db.refresh(new_mapping)\n return new_mapping\n\n\n# [/DEF:create_ad_mapping:Function]\n\n# [/DEF:AdminApi:Module]\n" + "body": "# [DEF:AdminApi:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: api, admin, users, roles, permissions\n# @PURPOSE: Admin API endpoints for user and role management.\n# @LAYER: API\n# @RELATION: [DEPENDS_ON] ->[AuthRepository:Class]\n# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function]\n# @RELATION: [DEPENDS_ON] ->[has_permission:Function]\n#\n# @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope.\n\n# [SECTION: IMPORTS]\nfrom typing import List\nfrom fastapi import APIRouter, Depends, HTTPException, status\nfrom sqlalchemy.orm import Session\nfrom ...core.database import get_auth_db\nfrom ...core.auth.repository import AuthRepository\nfrom ...core.auth.security import get_password_hash\nfrom ...schemas.auth import (\n User as UserSchema,\n UserCreate,\n UserUpdate,\n RoleSchema,\n RoleCreate,\n RoleUpdate,\n PermissionSchema,\n ADGroupMappingSchema,\n ADGroupMappingCreate,\n)\nfrom ...models.auth import User, Role, ADGroupMapping\nfrom ...dependencies import has_permission, get_plugin_loader\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.logger import logger, belief_scope\n\nlog = MarkerLogger(\"AdminApi\")\nfrom ...services.rbac_permission_catalog import (\n discover_declared_permissions,\n sync_permission_catalog,\n)\n# [/SECTION]\n\n# [DEF:router:Variable]\n# @RELATION: DEPENDS_ON -> fastapi.APIRouter\n# @PURPOSE: APIRouter instance for admin routes.\nrouter = APIRouter(prefix=\"/api/admin\", tags=[\"admin\"])\n# [/DEF:router:Variable]\n\n\n# [DEF:list_users:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all registered users.\n# @PRE: Current user has 'Admin' role.\n# @POST: Returns a list of UserSchema objects.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: List[UserSchema] - List of users.\n# @RELATION: CALLS -> User\n@router.get(\"/users\", response_model=List[UserSchema])\nasync def list_users(\n db: Session = Depends(get_auth_db), _=Depends(has_permission(\"admin:users\", \"READ\"))\n):\n with belief_scope(\"api.admin.list_users\"):\n users = db.query(User).all()\n return users\n\n\n# [/DEF:list_users:Function]\n\n\n# [DEF:create_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Creates a new local user.\n# @PRE: Current user has 'Admin' role.\n# @POST: New user is created in the database.\n# @PARAM: user_in (UserCreate) - New user data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: UserSchema - The created user.\n# @RELATION: [CALLS] ->[AuthRepository:Class]\n@router.post(\"/users\", response_model=UserSchema, status_code=status.HTTP_201_CREATED)\nasync def create_user(\n user_in: UserCreate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.create_user\"):\n repo = AuthRepository(db)\n if repo.get_user_by_username(user_in.username):\n raise HTTPException(status_code=400, detail=\"Username already exists\")\n\n new_user = User(\n username=user_in.username,\n email=user_in.email,\n password_hash=get_password_hash(user_in.password),\n auth_source=\"LOCAL\",\n is_active=user_in.is_active,\n )\n\n for role_name in user_in.roles:\n role = repo.get_role_by_name(role_name)\n if role:\n new_user.roles.append(role)\n\n db.add(new_user)\n db.commit()\n db.refresh(new_user)\n return new_user\n\n\n# [/DEF:create_user:Function]\n\n\n# [DEF:update_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Updates an existing user.\n# @PRE: Current user has 'Admin' role.\n# @POST: User record is updated in the database.\n# @PARAM: user_id (str) - Target user UUID.\n# @PARAM: user_in (UserUpdate) - Updated user data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: UserSchema - The updated user profile.\n# @RELATION: CALLS -> AuthRepository\n@router.put(\"/users/{user_id}\", response_model=UserSchema)\nasync def update_user(\n user_id: str,\n user_in: UserUpdate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.update_user\"):\n repo = AuthRepository(db)\n user = repo.get_user_by_id(user_id)\n if not user:\n raise HTTPException(status_code=404, detail=\"User not found\")\n\n if user_in.email is not None:\n user.email = user_in.email\n if user_in.is_active is not None:\n user.is_active = user_in.is_active\n if user_in.password is not None:\n user.password_hash = get_password_hash(user_in.password)\n\n if user_in.roles is not None:\n user.roles = []\n for role_name in user_in.roles:\n role = repo.get_role_by_name(role_name)\n if role:\n user.roles.append(role)\n\n db.commit()\n db.refresh(user)\n return user\n\n\n# [/DEF:update_user:Function]\n\n\n# [DEF:delete_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Deletes a user.\n# @PRE: Current user has 'Admin' role.\n# @POST: User record is removed from the database.\n# @PARAM: user_id (str) - Target user UUID.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: None\n# @RELATION: CALLS -> AuthRepository\n@router.delete(\"/users/{user_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_user(\n user_id: str,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.delete_user\"):\n log.reason(f\"Attempting to delete user {user_id}\")\n repo = AuthRepository(db)\n user = repo.get_user_by_id(user_id)\n if not user:\n log.explore(f\"User not found for deletion: {user_id}\", error=\"User not found\")\n raise HTTPException(status_code=404, detail=\"User not found\")\n\n log.reason(f\"Found user to delete: {user.username}\")\n db.delete(user)\n db.commit()\n log.reflect(f\"Successfully deleted user {user_id}\")\n return None\n\n\n# [/DEF:delete_user:Function]\n\n\n# [DEF:list_roles:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all available roles.\n# @RETURN: List[RoleSchema] - List of roles.\n# @RELATION: [CALLS] ->[Role:Class]\n@router.get(\"/roles\", response_model=List[RoleSchema])\nasync def list_roles(\n db: Session = Depends(get_auth_db), _=Depends(has_permission(\"admin:roles\", \"READ\"))\n):\n with belief_scope(\"api.admin.list_roles\"):\n return db.query(Role).all()\n\n\n# [/DEF:list_roles:Function]\n\n\n# [DEF:create_role:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Creates a new system role with associated permissions.\n# @PRE: Role name must be unique.\n# @POST: New Role record is created in auth.db.\n# @PARAM: role_in (RoleCreate) - New role data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: RoleSchema - The created role.\n# @SIDE_EFFECT: Commits new role and associations to auth.db.\n# @RELATION: [CALLS] ->[get_permission_by_id:Function]\n@router.post(\"/roles\", response_model=RoleSchema, status_code=status.HTTP_201_CREATED)\nasync def create_role(\n role_in: RoleCreate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:roles\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.create_role\"):\n if db.query(Role).filter(Role.name == role_in.name).first():\n raise HTTPException(status_code=400, detail=\"Role already exists\")\n\n new_role = Role(name=role_in.name, description=role_in.description)\n repo = AuthRepository(db)\n\n for perm_id_or_str in role_in.permissions:\n perm = repo.get_permission_by_id(perm_id_or_str)\n if not perm and \":\" in perm_id_or_str:\n res, act = perm_id_or_str.split(\":\", 1)\n perm = repo.get_permission_by_resource_action(res, act)\n\n if perm:\n new_role.permissions.append(perm)\n\n db.add(new_role)\n db.commit()\n db.refresh(new_role)\n return new_role\n\n\n# [/DEF:create_role:Function]\n\n\n# [DEF:update_role:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Updates an existing role's metadata and permissions.\n# @PRE: role_id must be a valid existing role UUID.\n# @POST: Role record is updated in auth.db.\n# @PARAM: role_id (str) - Target role identifier.\n# @PARAM: role_in (RoleUpdate) - Updated role data.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: RoleSchema - The updated role.\n# @SIDE_EFFECT: Commits updates to auth.db.\n# @RELATION: [CALLS] ->[get_role_by_id:Function]\n@router.put(\"/roles/{role_id}\", response_model=RoleSchema)\nasync def update_role(\n role_id: str,\n role_in: RoleUpdate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:roles\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.update_role\"):\n repo = AuthRepository(db)\n role = repo.get_role_by_id(role_id)\n if not role:\n raise HTTPException(status_code=404, detail=\"Role not found\")\n\n if role_in.name is not None:\n role.name = role_in.name\n if role_in.description is not None:\n role.description = role_in.description\n\n if role_in.permissions is not None:\n role.permissions = []\n for perm_id_or_str in role_in.permissions:\n perm = repo.get_permission_by_id(perm_id_or_str)\n if not perm and \":\" in perm_id_or_str:\n res, act = perm_id_or_str.split(\":\", 1)\n perm = repo.get_permission_by_resource_action(res, act)\n\n if perm:\n role.permissions.append(perm)\n\n db.commit()\n db.refresh(role)\n return role\n\n\n# [/DEF:update_role:Function]\n\n\n# [DEF:delete_role:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Removes a role from the system.\n# @PRE: role_id must be a valid existing role UUID.\n# @POST: Role record is removed from auth.db.\n# @PARAM: role_id (str) - Target role identifier.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: None\n# @SIDE_EFFECT: Deletes record from auth.db and commits.\n# @RELATION: [CALLS] ->[get_role_by_id:Function]\n@router.delete(\"/roles/{role_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_role(\n role_id: str,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:roles\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.delete_role\"):\n repo = AuthRepository(db)\n role = repo.get_role_by_id(role_id)\n if not role:\n raise HTTPException(status_code=404, detail=\"Role not found\")\n\n db.delete(role)\n db.commit()\n return None\n\n\n# [/DEF:delete_role:Function]\n\n\n# [DEF:list_permissions:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all available system permissions for assignment.\n# @POST: Returns a list of all PermissionSchema objects.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: List[PermissionSchema] - List of permissions.\n# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.list_permissions\n@router.get(\"/permissions\", response_model=List[PermissionSchema])\nasync def list_permissions(\n db: Session = Depends(get_auth_db),\n plugin_loader=Depends(get_plugin_loader),\n _=Depends(has_permission(\"admin:roles\", \"READ\")),\n):\n with belief_scope(\"api.admin.list_permissions\"):\n declared_permissions = discover_declared_permissions(\n plugin_loader=plugin_loader\n )\n inserted_count = sync_permission_catalog(\n db=db, declared_permissions=declared_permissions\n )\n if inserted_count > 0:\n log.reason(f\"Synchronized {inserted_count} missing RBAC permissions into auth catalog\")\n\n repo = AuthRepository(db)\n return repo.list_permissions()\n\n\n# [/DEF:list_permissions:Function]\n\n\n# [DEF:list_ad_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lists all AD Group to Role mappings.\n# @RELATION: CALLS -> ADGroupMapping\n@router.get(\"/ad-mappings\", response_model=List[ADGroupMappingSchema])\nasync def list_ad_mappings(\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"api.admin.list_ad_mappings\"):\n return db.query(ADGroupMapping).all()\n\n\n# [/DEF:list_ad_mappings:Function]\n\n\n# [DEF:create_ad_mapping:Function]\n# @RELATION: [DEPENDS_ON] ->[ADGroupMapping:Class]\n# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function]\n# @RELATION: [DEPENDS_ON] ->[has_permission:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Creates a new AD Group mapping.\n@router.post(\"/ad-mappings\", response_model=ADGroupMappingSchema)\nasync def create_ad_mapping(\n mapping_in: ADGroupMappingCreate,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.create_ad_mapping\"):\n new_mapping = ADGroupMapping(\n ad_group=mapping_in.ad_group, role_id=mapping_in.role_id\n )\n db.add(new_mapping)\n db.commit()\n db.refresh(new_mapping)\n return new_mapping\n\n\n# [/DEF:create_ad_mapping:Function]\n\n# [/DEF:AdminApi:Module]\n" }, { "contract_id": "list_users", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 47, - "end_line": 64, + "start_line": 50, + "end_line": 67, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8215,8 +8215,8 @@ "contract_id": "create_user", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 67, - "end_line": 106, + "start_line": 70, + "end_line": 109, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8291,8 +8291,8 @@ "contract_id": "update_user", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 109, - "end_line": 151, + "start_line": 112, + "end_line": 154, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8350,8 +8350,8 @@ "contract_id": "delete_user", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 154, - "end_line": 192, + "start_line": 157, + "end_line": 187, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8403,14 +8403,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:delete_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Deletes a user.\n# @PRE: Current user has 'Admin' role.\n# @POST: User record is removed from the database.\n# @PARAM: user_id (str) - Target user UUID.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: None\n# @RELATION: CALLS -> AuthRepository\n@router.delete(\"/users/{user_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_user(\n user_id: str,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.delete_user\"):\n logger.info(\n f\"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}\"\n )\n repo = AuthRepository(db)\n user = repo.get_user_by_id(user_id)\n if not user:\n logger.warning(\n f\"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}\"\n )\n raise HTTPException(status_code=404, detail=\"User not found\")\n\n logger.info(\n f\"[DEBUG] Found user to delete context={{'username': '{user.username}'}}\"\n )\n db.delete(user)\n db.commit()\n logger.info(\n f\"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}\"\n )\n return None\n\n\n# [/DEF:delete_user:Function]\n" + "body": "# [DEF:delete_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Deletes a user.\n# @PRE: Current user has 'Admin' role.\n# @POST: User record is removed from the database.\n# @PARAM: user_id (str) - Target user UUID.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: None\n# @RELATION: CALLS -> AuthRepository\n@router.delete(\"/users/{user_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_user(\n user_id: str,\n db: Session = Depends(get_auth_db),\n _=Depends(has_permission(\"admin:users\", \"WRITE\")),\n):\n with belief_scope(\"api.admin.delete_user\"):\n log.reason(f\"Attempting to delete user {user_id}\")\n repo = AuthRepository(db)\n user = repo.get_user_by_id(user_id)\n if not user:\n log.explore(f\"User not found for deletion: {user_id}\", error=\"User not found\")\n raise HTTPException(status_code=404, detail=\"User not found\")\n\n log.reason(f\"Found user to delete: {user.username}\")\n db.delete(user)\n db.commit()\n log.reflect(f\"Successfully deleted user {user_id}\")\n return None\n\n\n# [/DEF:delete_user:Function]\n" }, { "contract_id": "list_roles", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 195, - "end_line": 208, + "start_line": 190, + "end_line": 203, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8458,8 +8458,8 @@ "contract_id": "create_role", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 211, - "end_line": 249, + "start_line": 206, + "end_line": 244, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8544,8 +8544,8 @@ "contract_id": "update_role", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 252, - "end_line": 297, + "start_line": 247, + "end_line": 292, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8630,8 +8630,8 @@ "contract_id": "delete_role", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 300, - "end_line": 327, + "start_line": 295, + "end_line": 322, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8716,8 +8716,8 @@ "contract_id": "list_ad_mappings", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 363, - "end_line": 376, + "start_line": 355, + "end_line": 368, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -8740,8 +8740,8 @@ "contract_id": "create_ad_mapping", "contract_type": "Function", "file_path": "backend/src/api/routes/admin.py", - "start_line": 379, - "end_line": 401, + "start_line": 371, + "end_line": 393, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -14132,7 +14132,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/dashboards/_listing_routes.py", "start_line": 1, - "end_line": 398, + "end_line": 394, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -14216,14 +14216,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:DashboardListingRoutes:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: api, dashboards, listing, routes\n# @PURPOSE: Dashboard listing route handler for Dashboard Hub.\n# @LAYER: API\n# @RELATION: DEPENDS_ON -> [DashboardRouter]\n# @RELATION: DEPENDS_ON -> [DashboardSchemas]\n# @RELATION: DEPENDS_ON -> [DashboardHelpers]\n# @RELATION: DEPENDS_ON -> [DashboardProjection]\n\n# [SECTION: IMPORTS]\nimport os\nfrom typing import List, Optional, Dict, Any, Literal\nfrom fastapi import Depends, HTTPException, Query\nfrom sqlalchemy.orm import Session\nfrom src.dependencies import (\n get_config_manager, get_task_manager, get_resource_service,\n get_current_user, has_permission,\n)\nfrom src.core.database import get_db\nfrom src.core.logger import logger, belief_scope\nfrom src.models.auth import User\nfrom src.services.profile_service import ProfileService\nfrom ._helpers import _normalize_filter_values, _dashboard_git_filter_value\nfrom ._projection import (\n _project_dashboard_response_items, _get_profile_filter_binding,\n _resolve_profile_actor_aliases, _matches_dashboard_actor_aliases,\n)\nfrom ._schemas import EffectiveProfileFilter, DashboardsResponse\nfrom ._router import router\n# [/SECTION]\n\n\n# [DEF:get_dashboards:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status\n# @PRE: env_id must be a valid environment ID\n# @PRE: page must be >= 1 if provided\n# @PRE: page_size must be between 1 and 100 if provided\n# @POST: Returns a list of dashboards with enhanced metadata and pagination info\n# @POST: Response includes pagination metadata (page, page_size, total, total_pages)\n# @POST: Response includes effective profile filter metadata for main dashboards page context\n# @PARAM: env_id (str) - The environment ID to fetch dashboards from\n# @PARAM: search (Optional[str]) - Filter by title/slug\n# @PARAM: page (Optional[int]) - Page number (default: 1)\n# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100)\n# @RETURN: DashboardsResponse - List of dashboards with status metadata\n# @RELATION: CALLS ->[get_dashboards_with_status]\n@router.get(\"\", response_model=DashboardsResponse)\nasync def get_dashboards(\n env_id: str,\n search: Optional[str] = None,\n page: int = 1,\n page_size: int = 10,\n page_context: Literal[\"dashboards_main\", \"other\"] = Query(\"dashboards_main\"),\n apply_profile_default: bool = Query(default=True),\n override_show_all: bool = Query(default=False),\n filter_title: Optional[List[str]] = Query(default=None),\n filter_git_status: Optional[List[str]] = Query(default=None),\n filter_llm_status: Optional[List[str]] = Query(default=None),\n filter_changed_on: Optional[List[str]] = Query(default=None),\n filter_actor: Optional[List[str]] = Query(default=None),\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n resource_service=Depends(get_resource_service),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\n \"get_dashboards\",\n f\"env_id={env_id}, search={search}, page={page}, page_size={page_size}, \"\n f\"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}\",\n ):\n if page < 1:\n logger.error(f\"[get_dashboards][Coherence:Failed] Invalid page: {page}\")\n raise HTTPException(status_code=400, detail=\"Page must be >= 1\")\n if page_size < 1 or page_size > 100:\n logger.error(f\"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}\")\n raise HTTPException(status_code=400, detail=\"Page size must be between 1 and 100\")\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n logger.error(f\"[get_dashboards][Coherence:Failed] Environment not found: {env_id}\")\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n bound_username: Optional[str] = None\n can_apply_profile_filter = False\n can_apply_slug_filter = False\n effective_profile_filter = EffectiveProfileFilter(\n applied=False,\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=None,\n match_logic=None,\n )\n profile_service: Optional[ProfileService] = None\n\n try:\n profile_service_module = getattr(ProfileService, \"__module__\", \"\")\n is_mock_db = db.__class__.__module__.startswith(\"unittest.mock\")\n use_profile_service = (not is_mock_db) or profile_service_module.startswith(\n \"unittest.mock\"\n )\n if use_profile_service:\n profile_service = ProfileService(db=db, config_manager=config_manager)\n profile_preference = _get_profile_filter_binding(\n profile_service, current_user\n )\n normalized_username = (\n str(profile_preference.get(\"superset_username_normalized\") or \"\")\n .strip()\n .lower()\n )\n raw_username = (\n str(profile_preference.get(\"superset_username\") or \"\")\n .strip()\n .lower()\n )\n bound_username = normalized_username or raw_username or None\n\n can_apply_profile_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_my_dashboards\", False))\n and bool(bound_username)\n )\n can_apply_slug_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_slug_dashboards\", True))\n )\n\n profile_match_logic = None\n if can_apply_profile_filter and can_apply_slug_filter:\n profile_match_logic = \"owners_or_modified_by+slug_only\"\n elif can_apply_profile_filter:\n profile_match_logic = \"owners_or_modified_by\"\n elif can_apply_slug_filter:\n profile_match_logic = \"slug_only\"\n\n effective_profile_filter = EffectiveProfileFilter(\n applied=bool(can_apply_profile_filter or can_apply_slug_filter),\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=bound_username if can_apply_profile_filter else None,\n match_logic=profile_match_logic,\n )\n except Exception as profile_error:\n logger.explore(\n f\"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}\"\n )\n\n try:\n all_tasks = task_manager.get_all_tasks()\n title_filters = _normalize_filter_values(filter_title)\n git_filters = _normalize_filter_values(filter_git_status)\n llm_filters = _normalize_filter_values(filter_llm_status)\n changed_on_filters = _normalize_filter_values(filter_changed_on)\n actor_filters = _normalize_filter_values(filter_actor)\n has_column_filters = any(\n (\n title_filters,\n git_filters,\n llm_filters,\n changed_on_filters,\n actor_filters,\n )\n )\n needs_full_scan = (\n has_column_filters\n or bool(can_apply_profile_filter)\n or bool(can_apply_slug_filter)\n )\n\n if not needs_full_scan:\n try:\n page_payload = (\n await resource_service.get_dashboards_page_with_status(\n env,\n all_tasks,\n page=page,\n page_size=page_size,\n search=search,\n include_git_status=False,\n require_slug=bool(can_apply_slug_filter),\n )\n )\n paginated_dashboards = page_payload[\"dashboards\"]\n total = page_payload[\"total\"]\n total_pages = page_payload[\"total_pages\"]\n except Exception as page_error:\n logger.warning(\n \"[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s\",\n page_error,\n )\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n )\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n total = len(dashboards)\n total_pages = (\n (total + page_size - 1) // page_size if total > 0 else 1\n )\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n else:\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n )\n\n if (\n can_apply_profile_filter\n and bound_username\n and profile_service is not None\n ):\n actor_aliases = _resolve_profile_actor_aliases(env, bound_username)\n if not actor_aliases:\n actor_aliases = [bound_username]\n logger.reason(\n \"[REASON] Applying profile actor filter \"\n f\"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, \"\n f\"dashboards_before={len(dashboards)})\"\n )\n filtered_dashboards: List[Dict[str, Any]] = []\n max_actor_samples = 15\n for index, dashboard in enumerate(dashboards):\n owners_value = dashboard.get(\"owners\")\n created_by_value = dashboard.get(\"created_by\")\n modified_by_value = dashboard.get(\"modified_by\")\n matches_actor = _matches_dashboard_actor_aliases(\n profile_service=profile_service,\n actor_aliases=actor_aliases,\n owners=owners_value,\n modified_by=modified_by_value,\n )\n if index < max_actor_samples:\n logger.reflect(\n \"[REFLECT] Profile actor filter sample \"\n f\"(env={env_id}, dashboard_id={dashboard.get('id')}, \"\n f\"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, \"\n f\"owners={owners_value!r}, created_by={created_by_value!r}, \"\n f\"modified_by={modified_by_value!r}, matches={matches_actor})\"\n )\n if matches_actor:\n filtered_dashboards.append(dashboard)\n\n logger.reflect(\n \"[REFLECT] Profile actor filter summary \"\n f\"(env={env_id}, bound_username={bound_username!r}, \"\n f\"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})\"\n )\n dashboards = filtered_dashboards\n\n if can_apply_slug_filter:\n dashboards = [\n dashboard\n for dashboard in dashboards\n if str(dashboard.get(\"slug\") or \"\").strip()\n ]\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool:\n title_value = str(dashboard.get(\"title\") or \"\").strip().lower()\n if title_filters and title_value not in title_filters:\n return False\n\n if git_filters:\n git_value = _dashboard_git_filter_value(dashboard)\n if git_value not in git_filters:\n return False\n\n llm_value = (\n str(\n (\n (dashboard.get(\"last_task\") or {}).get(\n \"validation_status\"\n )\n )\n or \"UNKNOWN\"\n )\n .strip()\n .lower()\n )\n if llm_filters and llm_value not in llm_filters:\n return False\n\n changed_on_raw = (\n str(dashboard.get(\"last_modified\") or \"\").strip().lower()\n )\n changed_on_prefix = (\n changed_on_raw[:10]\n if len(changed_on_raw) >= 10\n else changed_on_raw\n )\n if (\n changed_on_filters\n and changed_on_raw not in changed_on_filters\n and changed_on_prefix not in changed_on_filters\n ):\n return False\n\n owners = dashboard.get(\"owners\") or []\n if isinstance(owners, list):\n actor_value = \", \".join(\n str(item).strip() for item in owners if str(item).strip()\n ).lower()\n else:\n actor_value = str(owners).strip().lower()\n if not actor_value:\n actor_value = \"-\"\n if actor_filters and actor_value not in actor_filters:\n return False\n return True\n\n if has_column_filters:\n dashboards = [\n d for d in dashboards if _matches_dashboard_filters(d)\n ]\n\n total = len(dashboards)\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n\n logger.info(\n f\"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards \"\n f\"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})\"\n )\n\n response_dashboards = _project_dashboard_response_items(\n paginated_dashboards\n )\n\n return DashboardsResponse(\n dashboards=response_dashboards,\n total=total,\n page=page,\n page_size=page_size,\n total_pages=total_pages,\n effective_profile_filter=effective_profile_filter,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(\n f\"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}\"\n )\n raise HTTPException(\n status_code=503, detail=f\"Failed to fetch dashboards: {str(e)}\"\n )\n\n\n# [/DEF:get_dashboards:Function]\n# [/DEF:DashboardListingRoutes:Module]\n" + "body": "# [DEF:DashboardListingRoutes:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: api, dashboards, listing, routes\n# @PURPOSE: Dashboard listing route handler for Dashboard Hub.\n# @LAYER: API\n# @RELATION: DEPENDS_ON -> [DashboardRouter]\n# @RELATION: DEPENDS_ON -> [DashboardSchemas]\n# @RELATION: DEPENDS_ON -> [DashboardHelpers]\n# @RELATION: DEPENDS_ON -> [DashboardProjection]\n\n# [SECTION: IMPORTS]\nimport os\nfrom typing import List, Optional, Dict, Any, Literal\nfrom fastapi import Depends, HTTPException, Query\nfrom sqlalchemy.orm import Session\nfrom src.dependencies import (\n get_config_manager, get_task_manager, get_resource_service,\n get_current_user, has_permission,\n)\nfrom src.core.database import get_db\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import logger, belief_scope\n\nlog = MarkerLogger(\"DashboardListing\")\nfrom src.models.auth import User\nfrom src.services.profile_service import ProfileService\nfrom ._helpers import _normalize_filter_values, _dashboard_git_filter_value\nfrom ._projection import (\n _project_dashboard_response_items, _get_profile_filter_binding,\n _resolve_profile_actor_aliases, _matches_dashboard_actor_aliases,\n)\nfrom ._schemas import EffectiveProfileFilter, DashboardsResponse\nfrom ._router import router\n# [/SECTION]\n\n\n# [DEF:get_dashboards:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status\n# @PRE: env_id must be a valid environment ID\n# @PRE: page must be >= 1 if provided\n# @PRE: page_size must be between 1 and 100 if provided\n# @POST: Returns a list of dashboards with enhanced metadata and pagination info\n# @POST: Response includes pagination metadata (page, page_size, total, total_pages)\n# @POST: Response includes effective profile filter metadata for main dashboards page context\n# @PARAM: env_id (str) - The environment ID to fetch dashboards from\n# @PARAM: search (Optional[str]) - Filter by title/slug\n# @PARAM: page (Optional[int]) - Page number (default: 1)\n# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100)\n# @RETURN: DashboardsResponse - List of dashboards with status metadata\n# @RELATION: CALLS ->[get_dashboards_with_status]\n@router.get(\"\", response_model=DashboardsResponse)\nasync def get_dashboards(\n env_id: str,\n search: Optional[str] = None,\n page: int = 1,\n page_size: int = 10,\n page_context: Literal[\"dashboards_main\", \"other\"] = Query(\"dashboards_main\"),\n apply_profile_default: bool = Query(default=True),\n override_show_all: bool = Query(default=False),\n filter_title: Optional[List[str]] = Query(default=None),\n filter_git_status: Optional[List[str]] = Query(default=None),\n filter_llm_status: Optional[List[str]] = Query(default=None),\n filter_changed_on: Optional[List[str]] = Query(default=None),\n filter_actor: Optional[List[str]] = Query(default=None),\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n resource_service=Depends(get_resource_service),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\n \"get_dashboards\",\n f\"env_id={env_id}, search={search}, page={page}, page_size={page_size}, \"\n f\"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}\",\n ):\n if page < 1:\n log.explore(f\"Invalid page: {page}\", error=\"Page must be >= 1\")\n raise HTTPException(status_code=400, detail=\"Page must be >= 1\")\n if page_size < 1 or page_size > 100:\n log.explore(f\"Invalid page_size: {page_size}\", error=\"Page size must be between 1 and 100\")\n raise HTTPException(status_code=400, detail=\"Page size must be between 1 and 100\")\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n log.explore(f\"Environment not found: {env_id}\", error=\"Environment not found\")\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n bound_username: Optional[str] = None\n can_apply_profile_filter = False\n can_apply_slug_filter = False\n effective_profile_filter = EffectiveProfileFilter(\n applied=False,\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=None,\n match_logic=None,\n )\n profile_service: Optional[ProfileService] = None\n\n try:\n profile_service_module = getattr(ProfileService, \"__module__\", \"\")\n is_mock_db = db.__class__.__module__.startswith(\"unittest.mock\")\n use_profile_service = (not is_mock_db) or profile_service_module.startswith(\n \"unittest.mock\"\n )\n if use_profile_service:\n profile_service = ProfileService(db=db, config_manager=config_manager)\n profile_preference = _get_profile_filter_binding(\n profile_service, current_user\n )\n normalized_username = (\n str(profile_preference.get(\"superset_username_normalized\") or \"\")\n .strip()\n .lower()\n )\n raw_username = (\n str(profile_preference.get(\"superset_username\") or \"\")\n .strip()\n .lower()\n )\n bound_username = normalized_username or raw_username or None\n\n can_apply_profile_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_my_dashboards\", False))\n and bool(bound_username)\n )\n can_apply_slug_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_slug_dashboards\", True))\n )\n\n profile_match_logic = None\n if can_apply_profile_filter and can_apply_slug_filter:\n profile_match_logic = \"owners_or_modified_by+slug_only\"\n elif can_apply_profile_filter:\n profile_match_logic = \"owners_or_modified_by\"\n elif can_apply_slug_filter:\n profile_match_logic = \"slug_only\"\n\n effective_profile_filter = EffectiveProfileFilter(\n applied=bool(can_apply_profile_filter or can_apply_slug_filter),\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=bound_username if can_apply_profile_filter else None,\n match_logic=profile_match_logic,\n )\n except Exception as profile_error:\n logger.explore(\n f\"Profile preference unavailable; continuing without profile-default filter\",\n error=str(profile_error),\n )\n\n try:\n all_tasks = task_manager.get_all_tasks()\n title_filters = _normalize_filter_values(filter_title)\n git_filters = _normalize_filter_values(filter_git_status)\n llm_filters = _normalize_filter_values(filter_llm_status)\n changed_on_filters = _normalize_filter_values(filter_changed_on)\n actor_filters = _normalize_filter_values(filter_actor)\n has_column_filters = any(\n (\n title_filters,\n git_filters,\n llm_filters,\n changed_on_filters,\n actor_filters,\n )\n )\n needs_full_scan = (\n has_column_filters\n or bool(can_apply_profile_filter)\n or bool(can_apply_slug_filter)\n )\n\n if not needs_full_scan:\n try:\n page_payload = (\n await resource_service.get_dashboards_page_with_status(\n env,\n all_tasks,\n page=page,\n page_size=page_size,\n search=search,\n include_git_status=False,\n require_slug=bool(can_apply_slug_filter),\n )\n )\n paginated_dashboards = page_payload[\"dashboards\"]\n total = page_payload[\"total\"]\n total_pages = page_payload[\"total_pages\"]\n except Exception as page_error:\n log.explore(f\"Page-based fetch failed; using compatibility fallback\", error=str(page_error))\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n )\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n total = len(dashboards)\n total_pages = (\n (total + page_size - 1) // page_size if total > 0 else 1\n )\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n else:\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n )\n\n if (\n can_apply_profile_filter\n and bound_username\n and profile_service is not None\n ):\n actor_aliases = _resolve_profile_actor_aliases(env, bound_username)\n if not actor_aliases:\n actor_aliases = [bound_username]\n logger.reason(\n f\"Applying profile actor filter \"\n f\"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, \"\n f\"dashboards_before={len(dashboards)})\"\n )\n filtered_dashboards: List[Dict[str, Any]] = []\n max_actor_samples = 15\n for index, dashboard in enumerate(dashboards):\n owners_value = dashboard.get(\"owners\")\n created_by_value = dashboard.get(\"created_by\")\n modified_by_value = dashboard.get(\"modified_by\")\n matches_actor = _matches_dashboard_actor_aliases(\n profile_service=profile_service,\n actor_aliases=actor_aliases,\n owners=owners_value,\n modified_by=modified_by_value,\n )\n if index < max_actor_samples:\n logger.reflect(\n f\"Profile actor filter sample \"\n f\"(env={env_id}, dashboard_id={dashboard.get('id')}, \"\n f\"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, \"\n f\"owners={owners_value!r}, created_by={created_by_value!r}, \"\n f\"modified_by={modified_by_value!r}, matches={matches_actor})\"\n )\n if matches_actor:\n filtered_dashboards.append(dashboard)\n\n logger.reflect(\n f\"Profile actor filter summary \"\n f\"(env={env_id}, bound_username={bound_username!r}, \"\n f\"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})\"\n )\n dashboards = filtered_dashboards\n\n if can_apply_slug_filter:\n dashboards = [\n dashboard\n for dashboard in dashboards\n if str(dashboard.get(\"slug\") or \"\").strip()\n ]\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool:\n title_value = str(dashboard.get(\"title\") or \"\").strip().lower()\n if title_filters and title_value not in title_filters:\n return False\n\n if git_filters:\n git_value = _dashboard_git_filter_value(dashboard)\n if git_value not in git_filters:\n return False\n\n llm_value = (\n str(\n (\n (dashboard.get(\"last_task\") or {}).get(\n \"validation_status\"\n )\n )\n or \"UNKNOWN\"\n )\n .strip()\n .lower()\n )\n if llm_filters and llm_value not in llm_filters:\n return False\n\n changed_on_raw = (\n str(dashboard.get(\"last_modified\") or \"\").strip().lower()\n )\n changed_on_prefix = (\n changed_on_raw[:10]\n if len(changed_on_raw) >= 10\n else changed_on_raw\n )\n if (\n changed_on_filters\n and changed_on_raw not in changed_on_filters\n and changed_on_prefix not in changed_on_filters\n ):\n return False\n\n owners = dashboard.get(\"owners\") or []\n if isinstance(owners, list):\n actor_value = \", \".join(\n str(item).strip() for item in owners if str(item).strip()\n ).lower()\n else:\n actor_value = str(owners).strip().lower()\n if not actor_value:\n actor_value = \"-\"\n if actor_filters and actor_value not in actor_filters:\n return False\n return True\n\n if has_column_filters:\n dashboards = [\n d for d in dashboards if _matches_dashboard_filters(d)\n ]\n\n total = len(dashboards)\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n\n log.reflect(f\"Returning {len(paginated_dashboards)} dashboards (page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})\")\n\n response_dashboards = _project_dashboard_response_items(\n paginated_dashboards\n )\n\n return DashboardsResponse(\n dashboards=response_dashboards,\n total=total,\n page=page,\n page_size=page_size,\n total_pages=total_pages,\n effective_profile_filter=effective_profile_filter,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n log.explore(\"Failed to fetch dashboards\", error=str(e))\n raise HTTPException(\n status_code=503, detail=f\"Failed to fetch dashboards: {str(e)}\"\n )\n\n\n# [/DEF:get_dashboards:Function]\n# [/DEF:DashboardListingRoutes:Module]\n" }, { "contract_id": "get_dashboards", "contract_type": "Function", "file_path": "backend/src/api/routes/dashboards/_listing_routes.py", - "start_line": 34, - "end_line": 397, + "start_line": 37, + "end_line": 393, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -14275,7 +14275,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_dashboards:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status\n# @PRE: env_id must be a valid environment ID\n# @PRE: page must be >= 1 if provided\n# @PRE: page_size must be between 1 and 100 if provided\n# @POST: Returns a list of dashboards with enhanced metadata and pagination info\n# @POST: Response includes pagination metadata (page, page_size, total, total_pages)\n# @POST: Response includes effective profile filter metadata for main dashboards page context\n# @PARAM: env_id (str) - The environment ID to fetch dashboards from\n# @PARAM: search (Optional[str]) - Filter by title/slug\n# @PARAM: page (Optional[int]) - Page number (default: 1)\n# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100)\n# @RETURN: DashboardsResponse - List of dashboards with status metadata\n# @RELATION: CALLS ->[get_dashboards_with_status]\n@router.get(\"\", response_model=DashboardsResponse)\nasync def get_dashboards(\n env_id: str,\n search: Optional[str] = None,\n page: int = 1,\n page_size: int = 10,\n page_context: Literal[\"dashboards_main\", \"other\"] = Query(\"dashboards_main\"),\n apply_profile_default: bool = Query(default=True),\n override_show_all: bool = Query(default=False),\n filter_title: Optional[List[str]] = Query(default=None),\n filter_git_status: Optional[List[str]] = Query(default=None),\n filter_llm_status: Optional[List[str]] = Query(default=None),\n filter_changed_on: Optional[List[str]] = Query(default=None),\n filter_actor: Optional[List[str]] = Query(default=None),\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n resource_service=Depends(get_resource_service),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\n \"get_dashboards\",\n f\"env_id={env_id}, search={search}, page={page}, page_size={page_size}, \"\n f\"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}\",\n ):\n if page < 1:\n logger.error(f\"[get_dashboards][Coherence:Failed] Invalid page: {page}\")\n raise HTTPException(status_code=400, detail=\"Page must be >= 1\")\n if page_size < 1 or page_size > 100:\n logger.error(f\"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}\")\n raise HTTPException(status_code=400, detail=\"Page size must be between 1 and 100\")\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n logger.error(f\"[get_dashboards][Coherence:Failed] Environment not found: {env_id}\")\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n bound_username: Optional[str] = None\n can_apply_profile_filter = False\n can_apply_slug_filter = False\n effective_profile_filter = EffectiveProfileFilter(\n applied=False,\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=None,\n match_logic=None,\n )\n profile_service: Optional[ProfileService] = None\n\n try:\n profile_service_module = getattr(ProfileService, \"__module__\", \"\")\n is_mock_db = db.__class__.__module__.startswith(\"unittest.mock\")\n use_profile_service = (not is_mock_db) or profile_service_module.startswith(\n \"unittest.mock\"\n )\n if use_profile_service:\n profile_service = ProfileService(db=db, config_manager=config_manager)\n profile_preference = _get_profile_filter_binding(\n profile_service, current_user\n )\n normalized_username = (\n str(profile_preference.get(\"superset_username_normalized\") or \"\")\n .strip()\n .lower()\n )\n raw_username = (\n str(profile_preference.get(\"superset_username\") or \"\")\n .strip()\n .lower()\n )\n bound_username = normalized_username or raw_username or None\n\n can_apply_profile_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_my_dashboards\", False))\n and bool(bound_username)\n )\n can_apply_slug_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_slug_dashboards\", True))\n )\n\n profile_match_logic = None\n if can_apply_profile_filter and can_apply_slug_filter:\n profile_match_logic = \"owners_or_modified_by+slug_only\"\n elif can_apply_profile_filter:\n profile_match_logic = \"owners_or_modified_by\"\n elif can_apply_slug_filter:\n profile_match_logic = \"slug_only\"\n\n effective_profile_filter = EffectiveProfileFilter(\n applied=bool(can_apply_profile_filter or can_apply_slug_filter),\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=bound_username if can_apply_profile_filter else None,\n match_logic=profile_match_logic,\n )\n except Exception as profile_error:\n logger.explore(\n f\"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}\"\n )\n\n try:\n all_tasks = task_manager.get_all_tasks()\n title_filters = _normalize_filter_values(filter_title)\n git_filters = _normalize_filter_values(filter_git_status)\n llm_filters = _normalize_filter_values(filter_llm_status)\n changed_on_filters = _normalize_filter_values(filter_changed_on)\n actor_filters = _normalize_filter_values(filter_actor)\n has_column_filters = any(\n (\n title_filters,\n git_filters,\n llm_filters,\n changed_on_filters,\n actor_filters,\n )\n )\n needs_full_scan = (\n has_column_filters\n or bool(can_apply_profile_filter)\n or bool(can_apply_slug_filter)\n )\n\n if not needs_full_scan:\n try:\n page_payload = (\n await resource_service.get_dashboards_page_with_status(\n env,\n all_tasks,\n page=page,\n page_size=page_size,\n search=search,\n include_git_status=False,\n require_slug=bool(can_apply_slug_filter),\n )\n )\n paginated_dashboards = page_payload[\"dashboards\"]\n total = page_payload[\"total\"]\n total_pages = page_payload[\"total_pages\"]\n except Exception as page_error:\n logger.warning(\n \"[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s\",\n page_error,\n )\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n )\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n total = len(dashboards)\n total_pages = (\n (total + page_size - 1) // page_size if total > 0 else 1\n )\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n else:\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n )\n\n if (\n can_apply_profile_filter\n and bound_username\n and profile_service is not None\n ):\n actor_aliases = _resolve_profile_actor_aliases(env, bound_username)\n if not actor_aliases:\n actor_aliases = [bound_username]\n logger.reason(\n \"[REASON] Applying profile actor filter \"\n f\"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, \"\n f\"dashboards_before={len(dashboards)})\"\n )\n filtered_dashboards: List[Dict[str, Any]] = []\n max_actor_samples = 15\n for index, dashboard in enumerate(dashboards):\n owners_value = dashboard.get(\"owners\")\n created_by_value = dashboard.get(\"created_by\")\n modified_by_value = dashboard.get(\"modified_by\")\n matches_actor = _matches_dashboard_actor_aliases(\n profile_service=profile_service,\n actor_aliases=actor_aliases,\n owners=owners_value,\n modified_by=modified_by_value,\n )\n if index < max_actor_samples:\n logger.reflect(\n \"[REFLECT] Profile actor filter sample \"\n f\"(env={env_id}, dashboard_id={dashboard.get('id')}, \"\n f\"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, \"\n f\"owners={owners_value!r}, created_by={created_by_value!r}, \"\n f\"modified_by={modified_by_value!r}, matches={matches_actor})\"\n )\n if matches_actor:\n filtered_dashboards.append(dashboard)\n\n logger.reflect(\n \"[REFLECT] Profile actor filter summary \"\n f\"(env={env_id}, bound_username={bound_username!r}, \"\n f\"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})\"\n )\n dashboards = filtered_dashboards\n\n if can_apply_slug_filter:\n dashboards = [\n dashboard\n for dashboard in dashboards\n if str(dashboard.get(\"slug\") or \"\").strip()\n ]\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool:\n title_value = str(dashboard.get(\"title\") or \"\").strip().lower()\n if title_filters and title_value not in title_filters:\n return False\n\n if git_filters:\n git_value = _dashboard_git_filter_value(dashboard)\n if git_value not in git_filters:\n return False\n\n llm_value = (\n str(\n (\n (dashboard.get(\"last_task\") or {}).get(\n \"validation_status\"\n )\n )\n or \"UNKNOWN\"\n )\n .strip()\n .lower()\n )\n if llm_filters and llm_value not in llm_filters:\n return False\n\n changed_on_raw = (\n str(dashboard.get(\"last_modified\") or \"\").strip().lower()\n )\n changed_on_prefix = (\n changed_on_raw[:10]\n if len(changed_on_raw) >= 10\n else changed_on_raw\n )\n if (\n changed_on_filters\n and changed_on_raw not in changed_on_filters\n and changed_on_prefix not in changed_on_filters\n ):\n return False\n\n owners = dashboard.get(\"owners\") or []\n if isinstance(owners, list):\n actor_value = \", \".join(\n str(item).strip() for item in owners if str(item).strip()\n ).lower()\n else:\n actor_value = str(owners).strip().lower()\n if not actor_value:\n actor_value = \"-\"\n if actor_filters and actor_value not in actor_filters:\n return False\n return True\n\n if has_column_filters:\n dashboards = [\n d for d in dashboards if _matches_dashboard_filters(d)\n ]\n\n total = len(dashboards)\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n\n logger.info(\n f\"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards \"\n f\"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})\"\n )\n\n response_dashboards = _project_dashboard_response_items(\n paginated_dashboards\n )\n\n return DashboardsResponse(\n dashboards=response_dashboards,\n total=total,\n page=page,\n page_size=page_size,\n total_pages=total_pages,\n effective_profile_filter=effective_profile_filter,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n logger.error(\n f\"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}\"\n )\n raise HTTPException(\n status_code=503, detail=f\"Failed to fetch dashboards: {str(e)}\"\n )\n\n\n# [/DEF:get_dashboards:Function]\n" + "body": "# [DEF:get_dashboards:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status\n# @PRE: env_id must be a valid environment ID\n# @PRE: page must be >= 1 if provided\n# @PRE: page_size must be between 1 and 100 if provided\n# @POST: Returns a list of dashboards with enhanced metadata and pagination info\n# @POST: Response includes pagination metadata (page, page_size, total, total_pages)\n# @POST: Response includes effective profile filter metadata for main dashboards page context\n# @PARAM: env_id (str) - The environment ID to fetch dashboards from\n# @PARAM: search (Optional[str]) - Filter by title/slug\n# @PARAM: page (Optional[int]) - Page number (default: 1)\n# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100)\n# @RETURN: DashboardsResponse - List of dashboards with status metadata\n# @RELATION: CALLS ->[get_dashboards_with_status]\n@router.get(\"\", response_model=DashboardsResponse)\nasync def get_dashboards(\n env_id: str,\n search: Optional[str] = None,\n page: int = 1,\n page_size: int = 10,\n page_context: Literal[\"dashboards_main\", \"other\"] = Query(\"dashboards_main\"),\n apply_profile_default: bool = Query(default=True),\n override_show_all: bool = Query(default=False),\n filter_title: Optional[List[str]] = Query(default=None),\n filter_git_status: Optional[List[str]] = Query(default=None),\n filter_llm_status: Optional[List[str]] = Query(default=None),\n filter_changed_on: Optional[List[str]] = Query(default=None),\n filter_actor: Optional[List[str]] = Query(default=None),\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n resource_service=Depends(get_resource_service),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\n \"get_dashboards\",\n f\"env_id={env_id}, search={search}, page={page}, page_size={page_size}, \"\n f\"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}\",\n ):\n if page < 1:\n log.explore(f\"Invalid page: {page}\", error=\"Page must be >= 1\")\n raise HTTPException(status_code=400, detail=\"Page must be >= 1\")\n if page_size < 1 or page_size > 100:\n log.explore(f\"Invalid page_size: {page_size}\", error=\"Page size must be between 1 and 100\")\n raise HTTPException(status_code=400, detail=\"Page size must be between 1 and 100\")\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n log.explore(f\"Environment not found: {env_id}\", error=\"Environment not found\")\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n bound_username: Optional[str] = None\n can_apply_profile_filter = False\n can_apply_slug_filter = False\n effective_profile_filter = EffectiveProfileFilter(\n applied=False,\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=None,\n match_logic=None,\n )\n profile_service: Optional[ProfileService] = None\n\n try:\n profile_service_module = getattr(ProfileService, \"__module__\", \"\")\n is_mock_db = db.__class__.__module__.startswith(\"unittest.mock\")\n use_profile_service = (not is_mock_db) or profile_service_module.startswith(\n \"unittest.mock\"\n )\n if use_profile_service:\n profile_service = ProfileService(db=db, config_manager=config_manager)\n profile_preference = _get_profile_filter_binding(\n profile_service, current_user\n )\n normalized_username = (\n str(profile_preference.get(\"superset_username_normalized\") or \"\")\n .strip()\n .lower()\n )\n raw_username = (\n str(profile_preference.get(\"superset_username\") or \"\")\n .strip()\n .lower()\n )\n bound_username = normalized_username or raw_username or None\n\n can_apply_profile_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_my_dashboards\", False))\n and bool(bound_username)\n )\n can_apply_slug_filter = (\n page_context == \"dashboards_main\"\n and bool(apply_profile_default)\n and not bool(override_show_all)\n and bool(profile_preference.get(\"show_only_slug_dashboards\", True))\n )\n\n profile_match_logic = None\n if can_apply_profile_filter and can_apply_slug_filter:\n profile_match_logic = \"owners_or_modified_by+slug_only\"\n elif can_apply_profile_filter:\n profile_match_logic = \"owners_or_modified_by\"\n elif can_apply_slug_filter:\n profile_match_logic = \"slug_only\"\n\n effective_profile_filter = EffectiveProfileFilter(\n applied=bool(can_apply_profile_filter or can_apply_slug_filter),\n source_page=page_context,\n override_show_all=bool(override_show_all),\n username=bound_username if can_apply_profile_filter else None,\n match_logic=profile_match_logic,\n )\n except Exception as profile_error:\n logger.explore(\n f\"Profile preference unavailable; continuing without profile-default filter\",\n error=str(profile_error),\n )\n\n try:\n all_tasks = task_manager.get_all_tasks()\n title_filters = _normalize_filter_values(filter_title)\n git_filters = _normalize_filter_values(filter_git_status)\n llm_filters = _normalize_filter_values(filter_llm_status)\n changed_on_filters = _normalize_filter_values(filter_changed_on)\n actor_filters = _normalize_filter_values(filter_actor)\n has_column_filters = any(\n (\n title_filters,\n git_filters,\n llm_filters,\n changed_on_filters,\n actor_filters,\n )\n )\n needs_full_scan = (\n has_column_filters\n or bool(can_apply_profile_filter)\n or bool(can_apply_slug_filter)\n )\n\n if not needs_full_scan:\n try:\n page_payload = (\n await resource_service.get_dashboards_page_with_status(\n env,\n all_tasks,\n page=page,\n page_size=page_size,\n search=search,\n include_git_status=False,\n require_slug=bool(can_apply_slug_filter),\n )\n )\n paginated_dashboards = page_payload[\"dashboards\"]\n total = page_payload[\"total\"]\n total_pages = page_payload[\"total_pages\"]\n except Exception as page_error:\n log.explore(f\"Page-based fetch failed; using compatibility fallback\", error=str(page_error))\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=False,\n )\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n total = len(dashboards)\n total_pages = (\n (total + page_size - 1) // page_size if total > 0 else 1\n )\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n else:\n if can_apply_slug_filter:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n require_slug=True,\n )\n else:\n dashboards = await resource_service.get_dashboards_with_status(\n env,\n all_tasks,\n include_git_status=bool(git_filters),\n )\n\n if (\n can_apply_profile_filter\n and bound_username\n and profile_service is not None\n ):\n actor_aliases = _resolve_profile_actor_aliases(env, bound_username)\n if not actor_aliases:\n actor_aliases = [bound_username]\n logger.reason(\n f\"Applying profile actor filter \"\n f\"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, \"\n f\"dashboards_before={len(dashboards)})\"\n )\n filtered_dashboards: List[Dict[str, Any]] = []\n max_actor_samples = 15\n for index, dashboard in enumerate(dashboards):\n owners_value = dashboard.get(\"owners\")\n created_by_value = dashboard.get(\"created_by\")\n modified_by_value = dashboard.get(\"modified_by\")\n matches_actor = _matches_dashboard_actor_aliases(\n profile_service=profile_service,\n actor_aliases=actor_aliases,\n owners=owners_value,\n modified_by=modified_by_value,\n )\n if index < max_actor_samples:\n logger.reflect(\n f\"Profile actor filter sample \"\n f\"(env={env_id}, dashboard_id={dashboard.get('id')}, \"\n f\"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, \"\n f\"owners={owners_value!r}, created_by={created_by_value!r}, \"\n f\"modified_by={modified_by_value!r}, matches={matches_actor})\"\n )\n if matches_actor:\n filtered_dashboards.append(dashboard)\n\n logger.reflect(\n f\"Profile actor filter summary \"\n f\"(env={env_id}, bound_username={bound_username!r}, \"\n f\"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})\"\n )\n dashboards = filtered_dashboards\n\n if can_apply_slug_filter:\n dashboards = [\n dashboard\n for dashboard in dashboards\n if str(dashboard.get(\"slug\") or \"\").strip()\n ]\n\n if search:\n search_lower = search.lower()\n dashboards = [\n d\n for d in dashboards\n if search_lower in d.get(\"title\", \"\").lower()\n or search_lower in d.get(\"slug\", \"\").lower()\n ]\n\n def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool:\n title_value = str(dashboard.get(\"title\") or \"\").strip().lower()\n if title_filters and title_value not in title_filters:\n return False\n\n if git_filters:\n git_value = _dashboard_git_filter_value(dashboard)\n if git_value not in git_filters:\n return False\n\n llm_value = (\n str(\n (\n (dashboard.get(\"last_task\") or {}).get(\n \"validation_status\"\n )\n )\n or \"UNKNOWN\"\n )\n .strip()\n .lower()\n )\n if llm_filters and llm_value not in llm_filters:\n return False\n\n changed_on_raw = (\n str(dashboard.get(\"last_modified\") or \"\").strip().lower()\n )\n changed_on_prefix = (\n changed_on_raw[:10]\n if len(changed_on_raw) >= 10\n else changed_on_raw\n )\n if (\n changed_on_filters\n and changed_on_raw not in changed_on_filters\n and changed_on_prefix not in changed_on_filters\n ):\n return False\n\n owners = dashboard.get(\"owners\") or []\n if isinstance(owners, list):\n actor_value = \", \".join(\n str(item).strip() for item in owners if str(item).strip()\n ).lower()\n else:\n actor_value = str(owners).strip().lower()\n if not actor_value:\n actor_value = \"-\"\n if actor_filters and actor_value not in actor_filters:\n return False\n return True\n\n if has_column_filters:\n dashboards = [\n d for d in dashboards if _matches_dashboard_filters(d)\n ]\n\n total = len(dashboards)\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n start_idx = (page - 1) * page_size\n end_idx = start_idx + page_size\n paginated_dashboards = dashboards[start_idx:end_idx]\n\n log.reflect(f\"Returning {len(paginated_dashboards)} dashboards (page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})\")\n\n response_dashboards = _project_dashboard_response_items(\n paginated_dashboards\n )\n\n return DashboardsResponse(\n dashboards=response_dashboards,\n total=total,\n page=page,\n page_size=page_size,\n total_pages=total_pages,\n effective_profile_filter=effective_profile_filter,\n )\n\n except HTTPException:\n raise\n except Exception as e:\n log.explore(\"Failed to fetch dashboards\", error=str(e))\n raise HTTPException(\n status_code=503, detail=f\"Failed to fetch dashboards: {str(e)}\"\n )\n\n\n# [/DEF:get_dashboards:Function]\n" }, { "contract_id": "DashboardProjection", @@ -15423,7 +15423,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", "start_line": 1, - "end_line": 806, + "end_line": 809, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -15449,14 +15449,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:DatasetReviewDependencies:Module]\n# @COMPLEXITY: 2\n# @PURPOSE: Dependency injection, serialization helpers, and feature-flag guards for dataset review endpoints.\n# @LAYER: API\n\nfrom __future__ import annotations\n\nimport json\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional, Union, cast\n\nfrom fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status\nfrom pydantic import BaseModel, Field\nfrom sqlalchemy.orm import Session\n\nfrom src.core.database import get_db\nfrom src.core.logger import belief_scope, logger\nfrom src.dependencies import (\n get_config_manager,\n get_current_user,\n get_task_manager,\n has_permission,\n)\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n AnswerKind,\n ApprovalState,\n ArtifactFormat,\n CandidateStatus,\n ClarificationSession,\n DatasetReviewSession,\n ExecutionMapping,\n FieldProvenance,\n MappingMethod,\n PreviewStatus,\n QuestionState,\n ReadinessState,\n RecommendedAction,\n SemanticCandidate,\n SemanticFieldEntry,\n SessionStatus,\n)\nfrom src.schemas.dataset_review import (\n ClarificationAnswerDto,\n ClarificationQuestionDto,\n ClarificationSessionDto,\n CompiledPreviewDto,\n DatasetRunContextDto,\n ExecutionMappingDto,\n SemanticFieldEntryDto,\n SessionDetail,\n SessionSummary,\n ValidationFindingDto,\n)\nfrom src.services.dataset_review.clarification_engine import (\n ClarificationAnswerCommand,\n ClarificationEngine,\n ClarificationQuestionPayload,\n ClarificationStateResult,\n)\nfrom src.services.dataset_review.orchestrator import (\n DatasetReviewOrchestrator,\n LaunchDatasetCommand,\n PreparePreviewCommand,\n StartSessionCommand,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionRepository,\n DatasetReviewSessionVersionConflictError,\n)\n\n\n# [DEF:StartSessionRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for starting one dataset review session.\nclass StartSessionRequest(BaseModel):\n source_kind: str = Field(..., pattern=\"^(superset_link|dataset_selection)$\")\n source_input: str = Field(..., min_length=1)\n environment_id: str = Field(..., min_length=1)\n\n\n# [/DEF:StartSessionRequest:Class]\n\n\n# [DEF:UpdateSessionRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for lifecycle state updates on an existing session.\nclass UpdateSessionRequest(BaseModel):\n status: SessionStatus\n note: Optional[str] = None\n\n\n# [/DEF:UpdateSessionRequest:Class]\n\n\n# [DEF:SessionCollectionResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Paginated session collection response.\nclass SessionCollectionResponse(BaseModel):\n items: List[SessionSummary]\n total: int\n page: int\n page_size: int\n has_next: bool\n\n\n# [/DEF:SessionCollectionResponse:Class]\n\n\n# [DEF:ExportArtifactResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Inline export response for documentation or validation outputs.\nclass ExportArtifactResponse(BaseModel):\n artifact_id: str\n session_id: str\n artifact_type: str\n format: str\n storage_ref: str\n created_by_user_id: str\n created_at: Optional[str] = None\n content: Dict[str, Any]\n\n\n# [/DEF:ExportArtifactResponse:Class]\n\n\n# [DEF:FieldSemanticUpdateRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for field-level semantic candidate acceptance or manual override.\nclass FieldSemanticUpdateRequest(BaseModel):\n candidate_id: Optional[str] = None\n verbose_name: Optional[str] = None\n description: Optional[str] = None\n display_format: Optional[str] = None\n lock_field: bool = False\n resolution_note: Optional[str] = None\n\n\n# [/DEF:FieldSemanticUpdateRequest:Class]\n\n\n# [DEF:FeedbackRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for thumbs up/down feedback.\nclass FeedbackRequest(BaseModel):\n feedback: str = Field(..., pattern=\"^(up|down)$\")\n\n\n# [/DEF:FeedbackRequest:Class]\n\n\n# [DEF:ClarificationAnswerRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for submitting one clarification answer.\nclass ClarificationAnswerRequest(BaseModel):\n question_id: str = Field(..., min_length=1)\n answer_kind: AnswerKind\n answer_value: Optional[str] = None\n\n\n# [/DEF:ClarificationAnswerRequest:Class]\n\n\n# [DEF:ClarificationSessionSummaryResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Summary DTO for current clarification session state.\nclass ClarificationSessionSummaryResponse(BaseModel):\n clarification_session_id: str\n session_id: str\n status: str\n current_question_id: Optional[str] = None\n resolved_count: int\n remaining_count: int\n summary_delta: Optional[str] = None\n\n\n# [/DEF:ClarificationSessionSummaryResponse:Class]\n\n\n# [DEF:ClarificationStateResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response DTO for current clarification state and active question payload.\nclass ClarificationStateResponse(BaseModel):\n clarification_session: Optional[ClarificationSessionSummaryResponse] = None\n current_question: Optional[ClarificationQuestionDto] = None\n\n\n# [/DEF:ClarificationStateResponse:Class]\n\n\n# [DEF:ClarificationAnswerResultResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response DTO for one clarification answer mutation result.\nclass ClarificationAnswerResultResponse(BaseModel):\n clarification_state: ClarificationStateResponse\n session: SessionSummary\n changed_findings: List[ValidationFindingDto]\n\n\n# [/DEF:ClarificationAnswerResultResponse:Class]\n\n\n# [DEF:FeedbackResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Minimal response DTO for persisted AI feedback actions.\nclass FeedbackResponse(BaseModel):\n target_id: str\n feedback: str\n\n\n# [/DEF:FeedbackResponse:Class]\n\n\n# [DEF:ApproveMappingRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Optional request DTO for explicit mapping approval audit notes.\nclass ApproveMappingRequest(BaseModel):\n approval_note: Optional[str] = None\n\n\n# [/DEF:ApproveMappingRequest:Class]\n\n\n# [DEF:BatchApproveSemanticItemRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for one batch semantic-approval item.\nclass BatchApproveSemanticItemRequest(BaseModel):\n field_id: str = Field(..., min_length=1)\n candidate_id: str = Field(..., min_length=1)\n lock_field: bool = False\n\n\n# [/DEF:BatchApproveSemanticItemRequest:Class]\n\n\n# [DEF:BatchApproveSemanticRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for explicit batch semantic approvals.\nclass BatchApproveSemanticRequest(BaseModel):\n items: List[BatchApproveSemanticItemRequest] = Field(..., min_length=1)\n\n\n# [/DEF:BatchApproveSemanticRequest:Class]\n\n\n# [DEF:BatchApproveMappingRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for explicit batch mapping approvals.\nclass BatchApproveMappingRequest(BaseModel):\n mapping_ids: List[str] = Field(..., min_length=1)\n approval_note: Optional[str] = None\n\n\n# [/DEF:BatchApproveMappingRequest:Class]\n\n\n# [DEF:PreviewEnqueueResultResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Async preview trigger response exposing only enqueue state.\nclass PreviewEnqueueResultResponse(BaseModel):\n session_id: str\n session_version: Optional[int] = None\n preview_status: str\n task_id: Optional[str] = None\n\n\n# [/DEF:PreviewEnqueueResultResponse:Class]\n\n\n# [DEF:MappingCollectionResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Wrapper for execution mapping list responses.\nclass MappingCollectionResponse(BaseModel):\n items: List[ExecutionMappingDto]\n\n\n# [/DEF:MappingCollectionResponse:Class]\n\n\n# [DEF:UpdateExecutionMappingRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for one manual execution-mapping override update.\nclass UpdateExecutionMappingRequest(BaseModel):\n effective_value: Optional[Any] = None\n mapping_method: Optional[str] = Field(default=None, pattern=\"^(manual_override|direct_match|heuristic_match|semantic_match)$\")\n transformation_note: Optional[str] = None\n\n\n# [/DEF:UpdateExecutionMappingRequest:Class]\n\n\n# [DEF:LaunchDatasetResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Launch result exposing audited run context and SQL Lab redirect target.\nclass LaunchDatasetResponse(BaseModel):\n run_context: DatasetRunContextDto\n redirect_url: str\n\n\n# [/DEF:LaunchDatasetResponse:Class]\n\n\n# --- Dependency Injection ---\n\n# [DEF:_require_auto_review_flag:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Guard US1 dataset review endpoints behind the configured feature flag.\ndef _require_auto_review_flag(config_manager=Depends(get_config_manager)) -> bool:\n with belief_scope(\"dataset_review.require_auto_review_flag\"):\n settings = config_manager.get_config().settings\n if not settings.features.dataset_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset review feature is disabled\")\n if not settings.ff_dataset_auto_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset auto review feature is disabled\")\n return True\n\n\n# [/DEF:_require_auto_review_flag:Function]\n\n\n# [DEF:_require_clarification_flag:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Guard clarification-specific US2 endpoints behind the configured feature flag.\ndef _require_clarification_flag(config_manager=Depends(get_config_manager)) -> bool:\n with belief_scope(\"dataset_review.require_clarification_flag\"):\n settings = config_manager.get_config().settings\n if not settings.features.dataset_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset review feature is disabled\")\n if not settings.ff_dataset_clarification:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset clarification feature is disabled\")\n return True\n\n\n# [/DEF:_require_clarification_flag:Function]\n\n\n# [DEF:_require_execution_flag:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Guard US3 execution endpoints behind the configured feature flag.\ndef _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool:\n with belief_scope(\"dataset_review.require_execution_flag\"):\n settings = config_manager.get_config().settings\n if not settings.features.dataset_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset review feature is disabled\")\n if not settings.ff_dataset_execution:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset execution feature is disabled\")\n return True\n\n\n# [/DEF:_require_execution_flag:Function]\n\n\n# [DEF:_get_repository:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build repository dependency.\ndef _get_repository(db: Session = Depends(get_db)) -> DatasetReviewSessionRepository:\n return DatasetReviewSessionRepository(db)\n\n\n# [/DEF:_get_repository:Function]\n\n\n# [DEF:_get_orchestrator:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build orchestrator dependency.\ndef _get_orchestrator(\n repository: DatasetReviewSessionRepository = Depends(_get_repository),\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n) -> DatasetReviewOrchestrator:\n return DatasetReviewOrchestrator(repository=repository, config_manager=config_manager, task_manager=task_manager)\n\n\n# [/DEF:_get_orchestrator:Function]\n\n\n# [DEF:_get_clarification_engine:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build clarification engine dependency.\ndef _get_clarification_engine(\n repository: DatasetReviewSessionRepository = Depends(_get_repository),\n) -> ClarificationEngine:\n return ClarificationEngine(repository=repository)\n\n\n# [/DEF:_get_clarification_engine:Function]\n\n\n# --- Serialization Helpers ---\n\n# [DEF:_serialize_session_summary:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map session aggregate into stable API summary DTO.\ndef _serialize_session_summary(session: DatasetReviewSession) -> SessionSummary:\n summary = SessionSummary.model_validate(session, from_attributes=True)\n summary.session_version = summary.version\n return summary\n\n\n# [/DEF:_serialize_session_summary:Function]\n\n\n# [DEF:_serialize_session_detail:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map session aggregate into stable API detail DTO.\ndef _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail:\n detail = SessionDetail.model_validate(session, from_attributes=True)\n detail.session_version = detail.version\n return detail\n\n\n# [/DEF:_serialize_session_detail:Function]\n\n\n# [DEF:_require_session_version_header:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Read the optimistic-lock session version header.\ndef _require_session_version_header(\n session_version: int = Header(..., alias=\"X-Session-Version\", ge=0),\n) -> int:\n return session_version\n\n\n# [/DEF:_require_session_version_header:Function]\n\n\n# [DEF:_build_session_version_conflict_http_exception:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Normalize optimistic-lock conflict errors into HTTP 409 responses.\ndef _build_session_version_conflict_http_exception(exc: DatasetReviewSessionVersionConflictError) -> HTTPException:\n return HTTPException(\n status_code=status.HTTP_409_CONFLICT,\n detail={\"error_code\": \"session_version_conflict\", \"message\": str(exc), \"session_id\": exc.session_id, \"expected_version\": exc.expected_version, \"actual_version\": exc.actual_version},\n )\n\n\n# [/DEF:_build_session_version_conflict_http_exception:Function]\n\n\n# [DEF:_enforce_session_version:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Convert repository optimistic-lock conflicts into deterministic HTTP 409 responses.\ndef _enforce_session_version(repository, session, expected_version):\n with belief_scope(\"_enforce_session_version\"):\n try:\n repository.require_session_version(session, expected_version)\n except DatasetReviewSessionVersionConflictError as exc:\n logger.explore(\"Dataset review optimistic-lock conflict detected\", extra={\"session_id\": exc.session_id, \"expected_version\": exc.expected_version, \"actual_version\": exc.actual_version})\n raise _build_session_version_conflict_http_exception(exc) from exc\n return session\n\n\n# [/DEF:_enforce_session_version:Function]\n\n\n# [DEF:_get_owned_session_or_404:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Resolve one session for current user or collaborator scope, returning 404 when inaccessible.\ndef _get_owned_session_or_404(repository, session_id, current_user):\n with belief_scope(\"_get_owned_session_or_404\"):\n session = repository.load_session_detail(session_id, current_user.id)\n if session is None:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Session not found\")\n return session\n\n\n# [/DEF:_get_owned_session_or_404:Function]\n\n\n# [DEF:_require_owner_mutation_scope:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Enforce owner-only mutation scope.\ndef _require_owner_mutation_scope(session, current_user):\n with belief_scope(\"_require_owner_mutation_scope\"):\n if session.user_id != current_user.id:\n raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=\"Only the owner can mutate dataset review state\")\n return session\n\n\n# [/DEF:_require_owner_mutation_scope:Function]\n\n\n# [DEF:_prepare_owned_session_mutation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Resolve owner-scoped mutation session and enforce optimistic-lock version.\ndef _prepare_owned_session_mutation(repository, session_id, current_user, expected_version):\n with belief_scope(\"_prepare_owned_session_mutation\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n _require_owner_mutation_scope(session, current_user)\n return _enforce_session_version(repository, session, expected_version)\n\n\n# [/DEF:_prepare_owned_session_mutation:Function]\n\n\n# [DEF:_commit_owned_session_mutation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Centralize session version bumping and commit semantics.\ndef _commit_owned_session_mutation(repository, session, *, refresh_targets=None):\n with belief_scope(\"_commit_owned_session_mutation\"):\n try:\n repository.commit_session_mutation(session, refresh_targets=refresh_targets)\n except DatasetReviewSessionVersionConflictError as exc:\n raise _build_session_version_conflict_http_exception(exc) from exc\n return session\n\n\n# [/DEF:_commit_owned_session_mutation:Function]\n\n\n# [DEF:_record_session_event:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Persist one explicit audit event for an owned mutation endpoint.\ndef _record_session_event(repository, session, current_user, *, event_type, event_summary, event_details=None):\n repository.event_logger.log_for_session(session, actor_user_id=current_user.id, event_type=event_type, event_summary=event_summary, event_details=event_details or {})\n\n\n# [/DEF:_record_session_event:Function]\n\n\n# [DEF:_serialize_semantic_field:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one semantic field into stable DTO.\ndef _serialize_semantic_field(field):\n payload = SemanticFieldEntryDto.model_validate(field, from_attributes=True)\n session_ref = getattr(field, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_semantic_field:Function]\n\n\n# [DEF:_serialize_execution_mapping:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one execution mapping into stable DTO.\ndef _serialize_execution_mapping(mapping):\n payload = ExecutionMappingDto.model_validate(mapping, from_attributes=True)\n session_ref = getattr(mapping, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_execution_mapping:Function]\n\n\n# [DEF:_serialize_preview:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one preview into stable DTO.\ndef _serialize_preview(preview, *, session_version_fallback=None):\n payload = CompiledPreviewDto.model_validate(preview, from_attributes=True)\n session_ref = getattr(preview, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n if version_value is None:\n version_value = session_version_fallback\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_preview:Function]\n\n\n# [DEF:_serialize_run_context:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one run context into stable DTO.\ndef _serialize_run_context(run_context):\n payload = DatasetRunContextDto.model_validate(run_context, from_attributes=True)\n session_ref = getattr(run_context, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_run_context:Function]\n\n\n# [DEF:_serialize_clarification_question_payload:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert clarification engine payload into API DTO.\ndef _serialize_clarification_question_payload(payload):\n if payload is None:\n return None\n return ClarificationQuestionDto.model_validate({\n \"question_id\": payload.question_id, \"clarification_session_id\": payload.clarification_session_id,\n \"topic_ref\": payload.topic_ref, \"question_text\": payload.question_text,\n \"why_it_matters\": payload.why_it_matters, \"current_guess\": payload.current_guess,\n \"priority\": payload.priority, \"state\": payload.state, \"options\": payload.options,\n \"answer\": None, \"created_at\": datetime.utcnow(), \"updated_at\": datetime.utcnow(),\n })\n\n\n# [/DEF:_serialize_clarification_question_payload:Function]\n\n\n# [DEF:_serialize_clarification_state:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert clarification engine state into API response.\ndef _serialize_clarification_state(state):\n return ClarificationStateResponse(\n clarification_session=ClarificationSessionSummaryResponse(\n clarification_session_id=state.clarification_session.clarification_session_id,\n session_id=state.clarification_session.session_id, status=state.clarification_session.status.value,\n current_question_id=state.clarification_session.current_question_id,\n resolved_count=state.clarification_session.resolved_count,\n remaining_count=state.clarification_session.remaining_count,\n summary_delta=state.clarification_session.summary_delta,\n ),\n current_question=_serialize_clarification_question_payload(state.current_question),\n )\n\n\n# [/DEF:_serialize_clarification_state:Function]\n\n\n# [DEF:_serialize_empty_clarification_state:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Return empty clarification payload.\ndef _serialize_empty_clarification_state():\n return ClarificationStateResponse(clarification_session=None, current_question=None)\n\n\n# [/DEF:_serialize_empty_clarification_state:Function]\n\n\n# [DEF:_get_latest_clarification_session_or_404:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Resolve the latest clarification aggregate or raise.\ndef _get_latest_clarification_session_or_404(session):\n if not session.clarification_sessions:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Clarification session not found\")\n return sorted(session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True)[0]\n\n\n# [/DEF:_get_latest_clarification_session_or_404:Function]\n\n\n# [DEF:_get_owned_mapping_or_404:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve one execution mapping inside one owned session.\ndef _get_owned_mapping_or_404(session, mapping_id):\n for mapping in session.execution_mappings:\n if mapping.mapping_id == mapping_id:\n return mapping\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Execution mapping not found\")\n\n\n# [/DEF:_get_owned_mapping_or_404:Function]\n\n\n# [DEF:_get_owned_field_or_404:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve a semantic field inside one owned session.\ndef _get_owned_field_or_404(session, field_id):\n for field in session.semantic_fields:\n if field.field_id == field_id:\n return field\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Semantic field not found\")\n\n\n# [/DEF:_get_owned_field_or_404:Function]\n\n\n# [DEF:_map_candidate_provenance:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Translate accepted semantic candidate type into stable field provenance.\ndef _map_candidate_provenance(candidate):\n if str(candidate.match_type.value) == \"exact\":\n return FieldProvenance.DICTIONARY_EXACT\n if str(candidate.match_type.value) == \"reference\":\n return FieldProvenance.REFERENCE_IMPORTED\n if str(candidate.match_type.value) == \"generated\":\n return FieldProvenance.AI_GENERATED\n return FieldProvenance.FUZZY_INFERRED\n\n\n# [/DEF:_map_candidate_provenance:Function]\n\n\n# [DEF:_resolve_candidate_source_version:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Resolve the semantic source version for one accepted candidate.\ndef _resolve_candidate_source_version(field, source_id):\n if not source_id:\n return None\n session = getattr(field, \"session\", None)\n if session is None:\n return None\n for source in getattr(session, \"semantic_sources\", []) or []:\n if source.source_id == source_id:\n return source.source_version\n return None\n\n\n# [/DEF:_resolve_candidate_source_version:Function]\n\n\n# [DEF:_update_semantic_field_state:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Apply field-level semantic manual override or candidate acceptance.\n# @POST: Manual overrides always set manual provenance plus lock.\ndef _update_semantic_field_state(field, request, changed_by):\n has_manual_override = any(v is not None for v in [request.verbose_name, request.description, request.display_format])\n selected_candidate = None\n if request.candidate_id:\n selected_candidate = next((c for c in field.candidates if c.candidate_id == request.candidate_id), None)\n if selected_candidate is None:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Semantic candidate not found\")\n\n if has_manual_override:\n field.verbose_name = request.verbose_name\n field.description = request.description\n field.display_format = request.display_format\n field.provenance = FieldProvenance.MANUAL_OVERRIDE\n field.source_id = None\n field.source_version = None\n field.confidence_rank = None\n field.is_locked = True\n field.has_conflict = False\n field.needs_review = False\n field.last_changed_by = changed_by\n for c in field.candidates:\n c.status = CandidateStatus.SUPERSEDED\n return field\n\n if selected_candidate is not None:\n field.verbose_name = selected_candidate.proposed_verbose_name\n field.description = selected_candidate.proposed_description\n field.display_format = selected_candidate.proposed_display_format\n field.provenance = _map_candidate_provenance(selected_candidate)\n field.source_id = selected_candidate.source_id\n field.source_version = _resolve_candidate_source_version(field, selected_candidate.source_id)\n field.confidence_rank = selected_candidate.candidate_rank\n field.is_locked = bool(request.lock_field or field.is_locked)\n field.has_conflict = len(field.candidates) > 1\n field.needs_review = False\n field.last_changed_by = changed_by\n for c in field.candidates:\n c.status = CandidateStatus.ACCEPTED if c.candidate_id == selected_candidate.candidate_id else CandidateStatus.SUPERSEDED\n return field\n\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=\"Provide candidate_id or at least one manual override field\")\n\n\n# [/DEF:_update_semantic_field_state:Function]\n\n\n# [DEF:_build_sql_lab_redirect_url:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build SQL Lab redirect URL.\ndef _build_sql_lab_redirect_url(environment_url, sql_lab_session_ref):\n base_url = str(environment_url or \"\").rstrip(\"/\")\n session_ref = str(sql_lab_session_ref or \"\").strip()\n if not base_url:\n raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=\"Superset environment URL is not configured\")\n if not session_ref:\n raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=\"SQL Lab session reference is missing\")\n return f\"{base_url}/superset/sqllab?queryId={session_ref}\"\n\n\n# [/DEF:_build_sql_lab_redirect_url:Function]\n\n\n# [DEF:_build_documentation_export:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Produce session documentation export content.\ndef _build_documentation_export(session, export_format):\n profile = session.profile\n findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code))\n if export_format == ArtifactFormat.MARKDOWN:\n lines = [f\"# Dataset Review: {session.dataset_ref}\", \"\", f\"- Session ID: {session.session_id}\", f\"- Environment: {session.environment_id}\", f\"- Readiness: {session.readiness_state.value}\", f\"- Recommended action: {session.recommended_action.value}\", \"\", \"## Business Summary\", profile.business_summary if profile else \"No profile summary available.\", \"\", \"## Findings\"]\n if findings:\n for f in findings:\n lines.append(f\"- [{f.severity.value}] {f.title}: {f.message}\")\n else:\n lines.append(\"- No findings recorded.\")\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/documentation.md\", \"content\": {\"markdown\": \"\\n\".join(lines)}}\n content = {\"session\": _serialize_session_summary(session).model_dump(mode=\"json\"), \"profile\": profile and {\"dataset_name\": profile.dataset_name, \"business_summary\": profile.business_summary, \"confidence_state\": profile.confidence_state.value, \"dataset_type\": profile.dataset_type}, \"findings\": [{\"code\": f.code, \"severity\": f.severity.value, \"title\": f.title, \"message\": f.message, \"resolution_state\": f.resolution_state.value} for f in findings]}\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/documentation.json\", \"content\": content}\n\n\n# [/DEF:_build_documentation_export:Function]\n\n\n# [DEF:_build_validation_export:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Produce validation-focused export content.\ndef _build_validation_export(session, export_format):\n findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code))\n if export_format == ArtifactFormat.MARKDOWN:\n lines = [f\"# Validation Report: {session.dataset_ref}\", \"\", f\"- Session ID: {session.session_id}\", f\"- Readiness: {session.readiness_state.value}\", \"\", \"## Findings\"]\n if findings:\n for f in findings:\n lines.append(f\"- `{f.code}` [{f.severity.value}] {f.message}\")\n else:\n lines.append(\"- No findings recorded.\")\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/validation.md\", \"content\": {\"markdown\": \"\\n\".join(lines)}}\n content = {\"session_id\": session.session_id, \"dataset_ref\": session.dataset_ref, \"readiness_state\": session.readiness_state.value, \"findings\": [{\"finding_id\": f.finding_id, \"area\": f.area.value, \"severity\": f.severity.value, \"code\": f.code, \"title\": f.title, \"message\": f.message, \"resolution_state\": f.resolution_state.value} for f in findings]}\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/validation.json\", \"content\": content}\n\n\n# [/DEF:_build_validation_export:Function]\n\n\n# [/DEF:DatasetReviewDependencies:Module]\n" + "body": "# [DEF:DatasetReviewDependencies:Module]\n# @COMPLEXITY: 2\n# @PURPOSE: Dependency injection, serialization helpers, and feature-flag guards for dataset review endpoints.\n# @LAYER: API\n\nfrom __future__ import annotations\n\nimport json\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional, Union, cast\n\nfrom fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status\nfrom pydantic import BaseModel, Field\nfrom sqlalchemy.orm import Session\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.database import get_db\nfrom src.core.logger import belief_scope\nfrom src.dependencies import (\n get_config_manager,\n get_current_user,\n get_task_manager,\n has_permission,\n)\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n AnswerKind,\n ApprovalState,\n ArtifactFormat,\n CandidateStatus,\n ClarificationSession,\n DatasetReviewSession,\n ExecutionMapping,\n FieldProvenance,\n MappingMethod,\n PreviewStatus,\n QuestionState,\n ReadinessState,\n RecommendedAction,\n SemanticCandidate,\n SemanticFieldEntry,\n SessionStatus,\n)\nfrom src.schemas.dataset_review import (\n ClarificationAnswerDto,\n ClarificationQuestionDto,\n ClarificationSessionDto,\n CompiledPreviewDto,\n DatasetRunContextDto,\n ExecutionMappingDto,\n SemanticFieldEntryDto,\n SessionDetail,\n SessionSummary,\n ValidationFindingDto,\n)\nfrom src.services.dataset_review.clarification_engine import (\n ClarificationAnswerCommand,\n ClarificationEngine,\n ClarificationQuestionPayload,\n ClarificationStateResult,\n)\nfrom src.services.dataset_review.orchestrator import (\n DatasetReviewOrchestrator,\n LaunchDatasetCommand,\n PreparePreviewCommand,\n StartSessionCommand,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionRepository,\n DatasetReviewSessionVersionConflictError,\n)\n\nlog = MarkerLogger(\"DatasetReviewDeps\")\n\n\n# [DEF:StartSessionRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for starting one dataset review session.\nclass StartSessionRequest(BaseModel):\n source_kind: str = Field(..., pattern=\"^(superset_link|dataset_selection)$\")\n source_input: str = Field(..., min_length=1)\n environment_id: str = Field(..., min_length=1)\n\n\n# [/DEF:StartSessionRequest:Class]\n\n\n# [DEF:UpdateSessionRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for lifecycle state updates on an existing session.\nclass UpdateSessionRequest(BaseModel):\n status: SessionStatus\n note: Optional[str] = None\n\n\n# [/DEF:UpdateSessionRequest:Class]\n\n\n# [DEF:SessionCollectionResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Paginated session collection response.\nclass SessionCollectionResponse(BaseModel):\n items: List[SessionSummary]\n total: int\n page: int\n page_size: int\n has_next: bool\n\n\n# [/DEF:SessionCollectionResponse:Class]\n\n\n# [DEF:ExportArtifactResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Inline export response for documentation or validation outputs.\nclass ExportArtifactResponse(BaseModel):\n artifact_id: str\n session_id: str\n artifact_type: str\n format: str\n storage_ref: str\n created_by_user_id: str\n created_at: Optional[str] = None\n content: Dict[str, Any]\n\n\n# [/DEF:ExportArtifactResponse:Class]\n\n\n# [DEF:FieldSemanticUpdateRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for field-level semantic candidate acceptance or manual override.\nclass FieldSemanticUpdateRequest(BaseModel):\n candidate_id: Optional[str] = None\n verbose_name: Optional[str] = None\n description: Optional[str] = None\n display_format: Optional[str] = None\n lock_field: bool = False\n resolution_note: Optional[str] = None\n\n\n# [/DEF:FieldSemanticUpdateRequest:Class]\n\n\n# [DEF:FeedbackRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for thumbs up/down feedback.\nclass FeedbackRequest(BaseModel):\n feedback: str = Field(..., pattern=\"^(up|down)$\")\n\n\n# [/DEF:FeedbackRequest:Class]\n\n\n# [DEF:ClarificationAnswerRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for submitting one clarification answer.\nclass ClarificationAnswerRequest(BaseModel):\n question_id: str = Field(..., min_length=1)\n answer_kind: AnswerKind\n answer_value: Optional[str] = None\n\n\n# [/DEF:ClarificationAnswerRequest:Class]\n\n\n# [DEF:ClarificationSessionSummaryResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Summary DTO for current clarification session state.\nclass ClarificationSessionSummaryResponse(BaseModel):\n clarification_session_id: str\n session_id: str\n status: str\n current_question_id: Optional[str] = None\n resolved_count: int\n remaining_count: int\n summary_delta: Optional[str] = None\n\n\n# [/DEF:ClarificationSessionSummaryResponse:Class]\n\n\n# [DEF:ClarificationStateResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response DTO for current clarification state and active question payload.\nclass ClarificationStateResponse(BaseModel):\n clarification_session: Optional[ClarificationSessionSummaryResponse] = None\n current_question: Optional[ClarificationQuestionDto] = None\n\n\n# [/DEF:ClarificationStateResponse:Class]\n\n\n# [DEF:ClarificationAnswerResultResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response DTO for one clarification answer mutation result.\nclass ClarificationAnswerResultResponse(BaseModel):\n clarification_state: ClarificationStateResponse\n session: SessionSummary\n changed_findings: List[ValidationFindingDto]\n\n\n# [/DEF:ClarificationAnswerResultResponse:Class]\n\n\n# [DEF:FeedbackResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Minimal response DTO for persisted AI feedback actions.\nclass FeedbackResponse(BaseModel):\n target_id: str\n feedback: str\n\n\n# [/DEF:FeedbackResponse:Class]\n\n\n# [DEF:ApproveMappingRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Optional request DTO for explicit mapping approval audit notes.\nclass ApproveMappingRequest(BaseModel):\n approval_note: Optional[str] = None\n\n\n# [/DEF:ApproveMappingRequest:Class]\n\n\n# [DEF:BatchApproveSemanticItemRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for one batch semantic-approval item.\nclass BatchApproveSemanticItemRequest(BaseModel):\n field_id: str = Field(..., min_length=1)\n candidate_id: str = Field(..., min_length=1)\n lock_field: bool = False\n\n\n# [/DEF:BatchApproveSemanticItemRequest:Class]\n\n\n# [DEF:BatchApproveSemanticRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for explicit batch semantic approvals.\nclass BatchApproveSemanticRequest(BaseModel):\n items: List[BatchApproveSemanticItemRequest] = Field(..., min_length=1)\n\n\n# [/DEF:BatchApproveSemanticRequest:Class]\n\n\n# [DEF:BatchApproveMappingRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for explicit batch mapping approvals.\nclass BatchApproveMappingRequest(BaseModel):\n mapping_ids: List[str] = Field(..., min_length=1)\n approval_note: Optional[str] = None\n\n\n# [/DEF:BatchApproveMappingRequest:Class]\n\n\n# [DEF:PreviewEnqueueResultResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Async preview trigger response exposing only enqueue state.\nclass PreviewEnqueueResultResponse(BaseModel):\n session_id: str\n session_version: Optional[int] = None\n preview_status: str\n task_id: Optional[str] = None\n\n\n# [/DEF:PreviewEnqueueResultResponse:Class]\n\n\n# [DEF:MappingCollectionResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Wrapper for execution mapping list responses.\nclass MappingCollectionResponse(BaseModel):\n items: List[ExecutionMappingDto]\n\n\n# [/DEF:MappingCollectionResponse:Class]\n\n\n# [DEF:UpdateExecutionMappingRequest:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Request DTO for one manual execution-mapping override update.\nclass UpdateExecutionMappingRequest(BaseModel):\n effective_value: Optional[Any] = None\n mapping_method: Optional[str] = Field(default=None, pattern=\"^(manual_override|direct_match|heuristic_match|semantic_match)$\")\n transformation_note: Optional[str] = None\n\n\n# [/DEF:UpdateExecutionMappingRequest:Class]\n\n\n# [DEF:LaunchDatasetResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Launch result exposing audited run context and SQL Lab redirect target.\nclass LaunchDatasetResponse(BaseModel):\n run_context: DatasetRunContextDto\n redirect_url: str\n\n\n# [/DEF:LaunchDatasetResponse:Class]\n\n\n# --- Dependency Injection ---\n\n# [DEF:_require_auto_review_flag:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Guard US1 dataset review endpoints behind the configured feature flag.\ndef _require_auto_review_flag(config_manager=Depends(get_config_manager)) -> bool:\n with belief_scope(\"dataset_review.require_auto_review_flag\"):\n settings = config_manager.get_config().settings\n if not settings.features.dataset_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset review feature is disabled\")\n if not settings.ff_dataset_auto_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset auto review feature is disabled\")\n return True\n\n\n# [/DEF:_require_auto_review_flag:Function]\n\n\n# [DEF:_require_clarification_flag:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Guard clarification-specific US2 endpoints behind the configured feature flag.\ndef _require_clarification_flag(config_manager=Depends(get_config_manager)) -> bool:\n with belief_scope(\"dataset_review.require_clarification_flag\"):\n settings = config_manager.get_config().settings\n if not settings.features.dataset_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset review feature is disabled\")\n if not settings.ff_dataset_clarification:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset clarification feature is disabled\")\n return True\n\n\n# [/DEF:_require_clarification_flag:Function]\n\n\n# [DEF:_require_execution_flag:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Guard US3 execution endpoints behind the configured feature flag.\ndef _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool:\n with belief_scope(\"dataset_review.require_execution_flag\"):\n settings = config_manager.get_config().settings\n if not settings.features.dataset_review:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset review feature is disabled\")\n if not settings.ff_dataset_execution:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Dataset execution feature is disabled\")\n return True\n\n\n# [/DEF:_require_execution_flag:Function]\n\n\n# [DEF:_get_repository:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build repository dependency.\ndef _get_repository(db: Session = Depends(get_db)) -> DatasetReviewSessionRepository:\n return DatasetReviewSessionRepository(db)\n\n\n# [/DEF:_get_repository:Function]\n\n\n# [DEF:_get_orchestrator:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build orchestrator dependency.\ndef _get_orchestrator(\n repository: DatasetReviewSessionRepository = Depends(_get_repository),\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n) -> DatasetReviewOrchestrator:\n return DatasetReviewOrchestrator(repository=repository, config_manager=config_manager, task_manager=task_manager)\n\n\n# [/DEF:_get_orchestrator:Function]\n\n\n# [DEF:_get_clarification_engine:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build clarification engine dependency.\ndef _get_clarification_engine(\n repository: DatasetReviewSessionRepository = Depends(_get_repository),\n) -> ClarificationEngine:\n return ClarificationEngine(repository=repository)\n\n\n# [/DEF:_get_clarification_engine:Function]\n\n\n# --- Serialization Helpers ---\n\n# [DEF:_serialize_session_summary:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map session aggregate into stable API summary DTO.\ndef _serialize_session_summary(session: DatasetReviewSession) -> SessionSummary:\n summary = SessionSummary.model_validate(session, from_attributes=True)\n summary.session_version = summary.version\n return summary\n\n\n# [/DEF:_serialize_session_summary:Function]\n\n\n# [DEF:_serialize_session_detail:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map session aggregate into stable API detail DTO.\ndef _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail:\n detail = SessionDetail.model_validate(session, from_attributes=True)\n detail.session_version = detail.version\n return detail\n\n\n# [/DEF:_serialize_session_detail:Function]\n\n\n# [DEF:_require_session_version_header:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Read the optimistic-lock session version header.\ndef _require_session_version_header(\n session_version: int = Header(..., alias=\"X-Session-Version\", ge=0),\n) -> int:\n return session_version\n\n\n# [/DEF:_require_session_version_header:Function]\n\n\n# [DEF:_build_session_version_conflict_http_exception:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Normalize optimistic-lock conflict errors into HTTP 409 responses.\ndef _build_session_version_conflict_http_exception(exc: DatasetReviewSessionVersionConflictError) -> HTTPException:\n return HTTPException(\n status_code=status.HTTP_409_CONFLICT,\n detail={\"error_code\": \"session_version_conflict\", \"message\": str(exc), \"session_id\": exc.session_id, \"expected_version\": exc.expected_version, \"actual_version\": exc.actual_version},\n )\n\n\n# [/DEF:_build_session_version_conflict_http_exception:Function]\n\n\n# [DEF:_enforce_session_version:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Convert repository optimistic-lock conflicts into deterministic HTTP 409 responses.\ndef _enforce_session_version(repository, session, expected_version):\n with belief_scope(\"_enforce_session_version\"):\n try:\n repository.require_session_version(session, expected_version)\n except DatasetReviewSessionVersionConflictError as exc:\n log.explore(\"Dataset review optimistic-lock conflict detected\", payload={\"session_id\": exc.session_id, \"expected_version\": exc.expected_version, \"actual_version\": exc.actual_version}, error=\"Optimistic lock conflict\")\n raise _build_session_version_conflict_http_exception(exc) from exc\n return session\n\n\n# [/DEF:_enforce_session_version:Function]\n\n\n# [DEF:_get_owned_session_or_404:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Resolve one session for current user or collaborator scope, returning 404 when inaccessible.\ndef _get_owned_session_or_404(repository, session_id, current_user):\n with belief_scope(\"_get_owned_session_or_404\"):\n session = repository.load_session_detail(session_id, current_user.id)\n if session is None:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Session not found\")\n return session\n\n\n# [/DEF:_get_owned_session_or_404:Function]\n\n\n# [DEF:_require_owner_mutation_scope:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Enforce owner-only mutation scope.\ndef _require_owner_mutation_scope(session, current_user):\n with belief_scope(\"_require_owner_mutation_scope\"):\n if session.user_id != current_user.id:\n raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=\"Only the owner can mutate dataset review state\")\n return session\n\n\n# [/DEF:_require_owner_mutation_scope:Function]\n\n\n# [DEF:_prepare_owned_session_mutation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Resolve owner-scoped mutation session and enforce optimistic-lock version.\ndef _prepare_owned_session_mutation(repository, session_id, current_user, expected_version):\n with belief_scope(\"_prepare_owned_session_mutation\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n _require_owner_mutation_scope(session, current_user)\n return _enforce_session_version(repository, session, expected_version)\n\n\n# [/DEF:_prepare_owned_session_mutation:Function]\n\n\n# [DEF:_commit_owned_session_mutation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Centralize session version bumping and commit semantics.\ndef _commit_owned_session_mutation(repository, session, *, refresh_targets=None):\n with belief_scope(\"_commit_owned_session_mutation\"):\n try:\n repository.commit_session_mutation(session, refresh_targets=refresh_targets)\n except DatasetReviewSessionVersionConflictError as exc:\n raise _build_session_version_conflict_http_exception(exc) from exc\n return session\n\n\n# [/DEF:_commit_owned_session_mutation:Function]\n\n\n# [DEF:_record_session_event:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Persist one explicit audit event for an owned mutation endpoint.\ndef _record_session_event(repository, session, current_user, *, event_type, event_summary, event_details=None):\n repository.event_logger.log_for_session(session, actor_user_id=current_user.id, event_type=event_type, event_summary=event_summary, event_details=event_details or {})\n\n\n# [/DEF:_record_session_event:Function]\n\n\n# [DEF:_serialize_semantic_field:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one semantic field into stable DTO.\ndef _serialize_semantic_field(field):\n payload = SemanticFieldEntryDto.model_validate(field, from_attributes=True)\n session_ref = getattr(field, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_semantic_field:Function]\n\n\n# [DEF:_serialize_execution_mapping:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one execution mapping into stable DTO.\ndef _serialize_execution_mapping(mapping):\n payload = ExecutionMappingDto.model_validate(mapping, from_attributes=True)\n session_ref = getattr(mapping, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_execution_mapping:Function]\n\n\n# [DEF:_serialize_preview:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one preview into stable DTO.\ndef _serialize_preview(preview, *, session_version_fallback=None):\n payload = CompiledPreviewDto.model_validate(preview, from_attributes=True)\n session_ref = getattr(preview, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n if version_value is None:\n version_value = session_version_fallback\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_preview:Function]\n\n\n# [DEF:_serialize_run_context:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Map one run context into stable DTO.\ndef _serialize_run_context(run_context):\n payload = DatasetRunContextDto.model_validate(run_context, from_attributes=True)\n session_ref = getattr(run_context, \"session\", None)\n version_value = getattr(session_ref, \"version\", None)\n payload.session_version = int(version_value or 0) if version_value is not None else None\n return payload\n\n\n# [/DEF:_serialize_run_context:Function]\n\n\n# [DEF:_serialize_clarification_question_payload:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert clarification engine payload into API DTO.\ndef _serialize_clarification_question_payload(payload):\n if payload is None:\n return None\n return ClarificationQuestionDto.model_validate({\n \"question_id\": payload.question_id, \"clarification_session_id\": payload.clarification_session_id,\n \"topic_ref\": payload.topic_ref, \"question_text\": payload.question_text,\n \"why_it_matters\": payload.why_it_matters, \"current_guess\": payload.current_guess,\n \"priority\": payload.priority, \"state\": payload.state, \"options\": payload.options,\n \"answer\": None, \"created_at\": datetime.utcnow(), \"updated_at\": datetime.utcnow(),\n })\n\n\n# [/DEF:_serialize_clarification_question_payload:Function]\n\n\n# [DEF:_serialize_clarification_state:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert clarification engine state into API response.\ndef _serialize_clarification_state(state):\n return ClarificationStateResponse(\n clarification_session=ClarificationSessionSummaryResponse(\n clarification_session_id=state.clarification_session.clarification_session_id,\n session_id=state.clarification_session.session_id, status=state.clarification_session.status.value,\n current_question_id=state.clarification_session.current_question_id,\n resolved_count=state.clarification_session.resolved_count,\n remaining_count=state.clarification_session.remaining_count,\n summary_delta=state.clarification_session.summary_delta,\n ),\n current_question=_serialize_clarification_question_payload(state.current_question),\n )\n\n\n# [/DEF:_serialize_clarification_state:Function]\n\n\n# [DEF:_serialize_empty_clarification_state:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Return empty clarification payload.\ndef _serialize_empty_clarification_state():\n return ClarificationStateResponse(clarification_session=None, current_question=None)\n\n\n# [/DEF:_serialize_empty_clarification_state:Function]\n\n\n# [DEF:_get_latest_clarification_session_or_404:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Resolve the latest clarification aggregate or raise.\ndef _get_latest_clarification_session_or_404(session):\n if not session.clarification_sessions:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Clarification session not found\")\n return sorted(session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True)[0]\n\n\n# [/DEF:_get_latest_clarification_session_or_404:Function]\n\n\n# [DEF:_get_owned_mapping_or_404:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve one execution mapping inside one owned session.\ndef _get_owned_mapping_or_404(session, mapping_id):\n for mapping in session.execution_mappings:\n if mapping.mapping_id == mapping_id:\n return mapping\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Execution mapping not found\")\n\n\n# [/DEF:_get_owned_mapping_or_404:Function]\n\n\n# [DEF:_get_owned_field_or_404:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve a semantic field inside one owned session.\ndef _get_owned_field_or_404(session, field_id):\n for field in session.semantic_fields:\n if field.field_id == field_id:\n return field\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Semantic field not found\")\n\n\n# [/DEF:_get_owned_field_or_404:Function]\n\n\n# [DEF:_map_candidate_provenance:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Translate accepted semantic candidate type into stable field provenance.\ndef _map_candidate_provenance(candidate):\n if str(candidate.match_type.value) == \"exact\":\n return FieldProvenance.DICTIONARY_EXACT\n if str(candidate.match_type.value) == \"reference\":\n return FieldProvenance.REFERENCE_IMPORTED\n if str(candidate.match_type.value) == \"generated\":\n return FieldProvenance.AI_GENERATED\n return FieldProvenance.FUZZY_INFERRED\n\n\n# [/DEF:_map_candidate_provenance:Function]\n\n\n# [DEF:_resolve_candidate_source_version:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Resolve the semantic source version for one accepted candidate.\ndef _resolve_candidate_source_version(field, source_id):\n if not source_id:\n return None\n session = getattr(field, \"session\", None)\n if session is None:\n return None\n for source in getattr(session, \"semantic_sources\", []) or []:\n if source.source_id == source_id:\n return source.source_version\n return None\n\n\n# [/DEF:_resolve_candidate_source_version:Function]\n\n\n# [DEF:_update_semantic_field_state:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Apply field-level semantic manual override or candidate acceptance.\n# @POST: Manual overrides always set manual provenance plus lock.\ndef _update_semantic_field_state(field, request, changed_by):\n has_manual_override = any(v is not None for v in [request.verbose_name, request.description, request.display_format])\n selected_candidate = None\n if request.candidate_id:\n selected_candidate = next((c for c in field.candidates if c.candidate_id == request.candidate_id), None)\n if selected_candidate is None:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=\"Semantic candidate not found\")\n\n if has_manual_override:\n field.verbose_name = request.verbose_name\n field.description = request.description\n field.display_format = request.display_format\n field.provenance = FieldProvenance.MANUAL_OVERRIDE\n field.source_id = None\n field.source_version = None\n field.confidence_rank = None\n field.is_locked = True\n field.has_conflict = False\n field.needs_review = False\n field.last_changed_by = changed_by\n for c in field.candidates:\n c.status = CandidateStatus.SUPERSEDED\n return field\n\n if selected_candidate is not None:\n field.verbose_name = selected_candidate.proposed_verbose_name\n field.description = selected_candidate.proposed_description\n field.display_format = selected_candidate.proposed_display_format\n field.provenance = _map_candidate_provenance(selected_candidate)\n field.source_id = selected_candidate.source_id\n field.source_version = _resolve_candidate_source_version(field, selected_candidate.source_id)\n field.confidence_rank = selected_candidate.candidate_rank\n field.is_locked = bool(request.lock_field or field.is_locked)\n field.has_conflict = len(field.candidates) > 1\n field.needs_review = False\n field.last_changed_by = changed_by\n for c in field.candidates:\n c.status = CandidateStatus.ACCEPTED if c.candidate_id == selected_candidate.candidate_id else CandidateStatus.SUPERSEDED\n return field\n\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=\"Provide candidate_id or at least one manual override field\")\n\n\n# [/DEF:_update_semantic_field_state:Function]\n\n\n# [DEF:_build_sql_lab_redirect_url:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build SQL Lab redirect URL.\ndef _build_sql_lab_redirect_url(environment_url, sql_lab_session_ref):\n base_url = str(environment_url or \"\").rstrip(\"/\")\n session_ref = str(sql_lab_session_ref or \"\").strip()\n if not base_url:\n raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=\"Superset environment URL is not configured\")\n if not session_ref:\n raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=\"SQL Lab session reference is missing\")\n return f\"{base_url}/superset/sqllab?queryId={session_ref}\"\n\n\n# [/DEF:_build_sql_lab_redirect_url:Function]\n\n\n# [DEF:_build_documentation_export:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Produce session documentation export content.\ndef _build_documentation_export(session, export_format):\n profile = session.profile\n findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code))\n if export_format == ArtifactFormat.MARKDOWN:\n lines = [f\"# Dataset Review: {session.dataset_ref}\", \"\", f\"- Session ID: {session.session_id}\", f\"- Environment: {session.environment_id}\", f\"- Readiness: {session.readiness_state.value}\", f\"- Recommended action: {session.recommended_action.value}\", \"\", \"## Business Summary\", profile.business_summary if profile else \"No profile summary available.\", \"\", \"## Findings\"]\n if findings:\n for f in findings:\n lines.append(f\"- [{f.severity.value}] {f.title}: {f.message}\")\n else:\n lines.append(\"- No findings recorded.\")\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/documentation.md\", \"content\": {\"markdown\": \"\\n\".join(lines)}}\n content = {\"session\": _serialize_session_summary(session).model_dump(mode=\"json\"), \"profile\": profile and {\"dataset_name\": profile.dataset_name, \"business_summary\": profile.business_summary, \"confidence_state\": profile.confidence_state.value, \"dataset_type\": profile.dataset_type}, \"findings\": [{\"code\": f.code, \"severity\": f.severity.value, \"title\": f.title, \"message\": f.message, \"resolution_state\": f.resolution_state.value} for f in findings]}\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/documentation.json\", \"content\": content}\n\n\n# [/DEF:_build_documentation_export:Function]\n\n\n# [DEF:_build_validation_export:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Produce validation-focused export content.\ndef _build_validation_export(session, export_format):\n findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code))\n if export_format == ArtifactFormat.MARKDOWN:\n lines = [f\"# Validation Report: {session.dataset_ref}\", \"\", f\"- Session ID: {session.session_id}\", f\"- Readiness: {session.readiness_state.value}\", \"\", \"## Findings\"]\n if findings:\n for f in findings:\n lines.append(f\"- `{f.code}` [{f.severity.value}] {f.message}\")\n else:\n lines.append(\"- No findings recorded.\")\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/validation.md\", \"content\": {\"markdown\": \"\\n\".join(lines)}}\n content = {\"session_id\": session.session_id, \"dataset_ref\": session.dataset_ref, \"readiness_state\": session.readiness_state.value, \"findings\": [{\"finding_id\": f.finding_id, \"area\": f.area.value, \"severity\": f.severity.value, \"code\": f.code, \"title\": f.title, \"message\": f.message, \"resolution_state\": f.resolution_state.value} for f in findings]}\n return {\"storage_ref\": f\"inline://dataset-review/{session.session_id}/validation.json\", \"content\": content}\n\n\n# [/DEF:_build_validation_export:Function]\n\n\n# [/DEF:DatasetReviewDependencies:Module]\n" }, { "contract_id": "StartSessionRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 73, - "end_line": 82, + "start_line": 76, + "end_line": 85, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15472,8 +15472,8 @@ "contract_id": "UpdateSessionRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 85, - "end_line": 93, + "start_line": 88, + "end_line": 96, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15489,8 +15489,8 @@ "contract_id": "SessionCollectionResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 96, - "end_line": 107, + "start_line": 99, + "end_line": 110, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15506,8 +15506,8 @@ "contract_id": "ExportArtifactResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 110, - "end_line": 124, + "start_line": 113, + "end_line": 127, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15523,8 +15523,8 @@ "contract_id": "FieldSemanticUpdateRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 127, - "end_line": 139, + "start_line": 130, + "end_line": 142, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15540,8 +15540,8 @@ "contract_id": "FeedbackRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 142, - "end_line": 149, + "start_line": 145, + "end_line": 152, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15557,8 +15557,8 @@ "contract_id": "ClarificationAnswerRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 152, - "end_line": 161, + "start_line": 155, + "end_line": 164, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15574,8 +15574,8 @@ "contract_id": "ClarificationSessionSummaryResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 164, - "end_line": 177, + "start_line": 167, + "end_line": 180, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15591,8 +15591,8 @@ "contract_id": "ClarificationStateResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 180, - "end_line": 188, + "start_line": 183, + "end_line": 191, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15608,8 +15608,8 @@ "contract_id": "ClarificationAnswerResultResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 191, - "end_line": 200, + "start_line": 194, + "end_line": 203, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15625,8 +15625,8 @@ "contract_id": "FeedbackResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 203, - "end_line": 211, + "start_line": 206, + "end_line": 214, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15642,8 +15642,8 @@ "contract_id": "ApproveMappingRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 214, - "end_line": 221, + "start_line": 217, + "end_line": 224, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15659,8 +15659,8 @@ "contract_id": "BatchApproveSemanticItemRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 224, - "end_line": 233, + "start_line": 227, + "end_line": 236, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15676,8 +15676,8 @@ "contract_id": "BatchApproveSemanticRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 236, - "end_line": 243, + "start_line": 239, + "end_line": 246, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15693,8 +15693,8 @@ "contract_id": "BatchApproveMappingRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 246, - "end_line": 254, + "start_line": 249, + "end_line": 257, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15710,8 +15710,8 @@ "contract_id": "PreviewEnqueueResultResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 257, - "end_line": 267, + "start_line": 260, + "end_line": 270, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15727,8 +15727,8 @@ "contract_id": "MappingCollectionResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 270, - "end_line": 277, + "start_line": 273, + "end_line": 280, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15744,8 +15744,8 @@ "contract_id": "UpdateExecutionMappingRequest", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 280, - "end_line": 289, + "start_line": 283, + "end_line": 292, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15761,8 +15761,8 @@ "contract_id": "LaunchDatasetResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 292, - "end_line": 300, + "start_line": 295, + "end_line": 303, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15778,8 +15778,8 @@ "contract_id": "_require_auto_review_flag", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 305, - "end_line": 318, + "start_line": 308, + "end_line": 321, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -15795,8 +15795,8 @@ "contract_id": "_require_clarification_flag", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 321, - "end_line": 334, + "start_line": 324, + "end_line": 337, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -15812,8 +15812,8 @@ "contract_id": "_require_execution_flag", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 337, - "end_line": 350, + "start_line": 340, + "end_line": 353, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -15829,8 +15829,8 @@ "contract_id": "_get_repository", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 353, - "end_line": 360, + "start_line": 356, + "end_line": 363, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15846,8 +15846,8 @@ "contract_id": "_get_orchestrator", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 363, - "end_line": 374, + "start_line": 366, + "end_line": 377, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15863,8 +15863,8 @@ "contract_id": "_get_clarification_engine", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 377, - "end_line": 386, + "start_line": 380, + "end_line": 389, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15880,8 +15880,8 @@ "contract_id": "_serialize_session_summary", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 391, - "end_line": 400, + "start_line": 394, + "end_line": 403, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15897,8 +15897,8 @@ "contract_id": "_serialize_session_detail", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 403, - "end_line": 412, + "start_line": 406, + "end_line": 415, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15914,8 +15914,8 @@ "contract_id": "_require_session_version_header", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 415, - "end_line": 424, + "start_line": 418, + "end_line": 427, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15931,8 +15931,8 @@ "contract_id": "_build_session_version_conflict_http_exception", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 427, - "end_line": 437, + "start_line": 430, + "end_line": 440, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -15948,8 +15948,8 @@ "contract_id": "_enforce_session_version", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 440, - "end_line": 453, + "start_line": 443, + "end_line": 456, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -15969,14 +15969,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:_enforce_session_version:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Convert repository optimistic-lock conflicts into deterministic HTTP 409 responses.\ndef _enforce_session_version(repository, session, expected_version):\n with belief_scope(\"_enforce_session_version\"):\n try:\n repository.require_session_version(session, expected_version)\n except DatasetReviewSessionVersionConflictError as exc:\n logger.explore(\"Dataset review optimistic-lock conflict detected\", extra={\"session_id\": exc.session_id, \"expected_version\": exc.expected_version, \"actual_version\": exc.actual_version})\n raise _build_session_version_conflict_http_exception(exc) from exc\n return session\n\n\n# [/DEF:_enforce_session_version:Function]\n" + "body": "# [DEF:_enforce_session_version:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Convert repository optimistic-lock conflicts into deterministic HTTP 409 responses.\ndef _enforce_session_version(repository, session, expected_version):\n with belief_scope(\"_enforce_session_version\"):\n try:\n repository.require_session_version(session, expected_version)\n except DatasetReviewSessionVersionConflictError as exc:\n log.explore(\"Dataset review optimistic-lock conflict detected\", payload={\"session_id\": exc.session_id, \"expected_version\": exc.expected_version, \"actual_version\": exc.actual_version}, error=\"Optimistic lock conflict\")\n raise _build_session_version_conflict_http_exception(exc) from exc\n return session\n\n\n# [/DEF:_enforce_session_version:Function]\n" }, { "contract_id": "_get_owned_session_or_404", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 456, - "end_line": 467, + "start_line": 459, + "end_line": 470, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -16002,8 +16002,8 @@ "contract_id": "_require_owner_mutation_scope", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 470, - "end_line": 480, + "start_line": 473, + "end_line": 483, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -16019,8 +16019,8 @@ "contract_id": "_prepare_owned_session_mutation", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 483, - "end_line": 493, + "start_line": 486, + "end_line": 496, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -16046,8 +16046,8 @@ "contract_id": "_commit_owned_session_mutation", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 496, - "end_line": 508, + "start_line": 499, + "end_line": 511, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -16073,8 +16073,8 @@ "contract_id": "_record_session_event", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 511, - "end_line": 518, + "start_line": 514, + "end_line": 521, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16090,8 +16090,8 @@ "contract_id": "_serialize_semantic_field", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 521, - "end_line": 532, + "start_line": 524, + "end_line": 535, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16107,8 +16107,8 @@ "contract_id": "_serialize_execution_mapping", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 535, - "end_line": 546, + "start_line": 538, + "end_line": 549, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16124,8 +16124,8 @@ "contract_id": "_serialize_preview", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 549, - "end_line": 562, + "start_line": 552, + "end_line": 565, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16141,8 +16141,8 @@ "contract_id": "_serialize_run_context", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 565, - "end_line": 576, + "start_line": 568, + "end_line": 579, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16158,8 +16158,8 @@ "contract_id": "_serialize_clarification_question_payload", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 579, - "end_line": 594, + "start_line": 582, + "end_line": 597, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16175,8 +16175,8 @@ "contract_id": "_serialize_clarification_state", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 597, - "end_line": 614, + "start_line": 600, + "end_line": 617, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16192,8 +16192,8 @@ "contract_id": "_serialize_empty_clarification_state", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 617, - "end_line": 624, + "start_line": 620, + "end_line": 627, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16209,8 +16209,8 @@ "contract_id": "_get_latest_clarification_session_or_404", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 627, - "end_line": 636, + "start_line": 630, + "end_line": 639, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16226,8 +16226,8 @@ "contract_id": "_get_owned_mapping_or_404", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 639, - "end_line": 649, + "start_line": 642, + "end_line": 652, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -16243,8 +16243,8 @@ "contract_id": "_get_owned_field_or_404", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 652, - "end_line": 662, + "start_line": 655, + "end_line": 665, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -16260,8 +16260,8 @@ "contract_id": "_map_candidate_provenance", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 665, - "end_line": 678, + "start_line": 668, + "end_line": 681, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16277,8 +16277,8 @@ "contract_id": "_resolve_candidate_source_version", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 681, - "end_line": 696, + "start_line": 684, + "end_line": 699, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16294,8 +16294,8 @@ "contract_id": "_update_semantic_field_state", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 699, - "end_line": 746, + "start_line": 702, + "end_line": 749, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -16331,8 +16331,8 @@ "contract_id": "_build_sql_lab_redirect_url", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 749, - "end_line": 762, + "start_line": 752, + "end_line": 765, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -16348,8 +16348,8 @@ "contract_id": "_build_documentation_export", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 765, - "end_line": 783, + "start_line": 768, + "end_line": 786, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -16365,8 +16365,8 @@ "contract_id": "_build_validation_export", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_dependencies.py", - "start_line": 786, - "end_line": 803, + "start_line": 789, + "end_line": 806, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -16412,14 +16412,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:DatasetReviewRoutes:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist thumbs up/down feedback for clarification question/answer content.\n\n\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import Any, List, Optional, Union, cast\n\nfrom fastapi import APIRouter, Depends, HTTPException, Query, Response, status\n\nfrom src.core.logger import belief_scope, logger\nfrom src.dependencies import get_config_manager, get_current_user, has_permission\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n ApprovalState,\n ArtifactFormat,\n FieldProvenance,\n MappingMethod,\n PreviewStatus,\n ReadinessState,\n RecommendedAction,\n SessionStatus,\n)\nfrom src.schemas.dataset_review import (\n ClarificationAnswerDto,\n CompiledPreviewDto,\n ExecutionMappingDto,\n SemanticFieldEntryDto,\n SessionSummary,\n ValidationFindingDto,\n)\nfrom src.services.dataset_review.clarification_engine import (\n ClarificationAnswerCommand,\n ClarificationStateResult,\n)\nfrom src.services.dataset_review.orchestrator import (\n LaunchDatasetCommand,\n PreparePreviewCommand,\n StartSessionCommand,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionVersionConflictError,\n)\nfrom src.api.routes.dataset_review_pkg._dependencies import (\n BatchApproveMappingRequest,\n BatchApproveSemanticRequest,\n ClarificationAnswerRequest,\n ClarificationAnswerResultResponse,\n ClarificationStateResponse,\n ExportArtifactResponse,\n FeedbackRequest,\n FeedbackResponse,\n FieldSemanticUpdateRequest,\n LaunchDatasetResponse,\n MappingCollectionResponse,\n PreviewEnqueueResultResponse,\n SessionCollectionResponse,\n StartSessionRequest,\n UpdateExecutionMappingRequest,\n UpdateSessionRequest,\n _build_documentation_export,\n _build_sql_lab_redirect_url,\n _build_validation_export,\n _commit_owned_session_mutation,\n _get_clarification_engine,\n _get_latest_clarification_session_or_404,\n _get_owned_field_or_404,\n _get_owned_mapping_or_404,\n _get_owned_session_or_404,\n _get_orchestrator,\n _get_repository,\n _prepare_owned_session_mutation,\n _record_session_event,\n _require_auto_review_flag,\n _require_clarification_flag,\n _require_execution_flag,\n _require_session_version_header,\n _serialize_clarification_state,\n _serialize_empty_clarification_state,\n _serialize_execution_mapping,\n _serialize_preview,\n _serialize_run_context,\n _serialize_semantic_field,\n _serialize_session_detail,\n _serialize_session_summary,\n _update_semantic_field_state,\n _build_session_version_conflict_http_exception,\n)\n\nrouter = APIRouter(prefix=\"/api/dataset-orchestration\", tags=[\"Dataset Orchestration\"])\n\n\n# [DEF:list_sessions:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: List resumable dataset review sessions for the current user.\n@router.get(\n \"/sessions\",\n response_model=SessionCollectionResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def list_sessions(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.list_sessions\"):\n logger.reason(\n \"Listing dataset review sessions\",\n extra={\"user_id\": current_user.id, \"page\": page, \"page_size\": page_size},\n )\n sessions = repository.list_sessions_for_user(current_user.id)\n start = (page - 1) * page_size\n end = start + page_size\n items = [_serialize_session_summary(s) for s in sessions[start:end]]\n logger.reflect(\n \"Session page assembled\",\n extra={\n \"user_id\": current_user.id,\n \"returned\": len(items),\n \"total\": len(sessions),\n },\n )\n return SessionCollectionResponse(\n items=items,\n total=len(sessions),\n page=page,\n page_size=page_size,\n has_next=end < len(sessions),\n )\n\n\n# [/DEF:list_sessions:Function]\n\n\n# [DEF:start_session:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Start a new dataset review session from a Superset link or dataset selection.\n@router.post(\n \"/sessions\",\n response_model=SessionSummary,\n status_code=status.HTTP_201_CREATED,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def start_session(\n request: StartSessionRequest,\n orchestrator=Depends(_get_orchestrator),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"start_session\"):\n logger.reason(\n \"Starting dataset review session\",\n extra={\n \"user_id\": current_user.id,\n \"environment_id\": request.environment_id,\n },\n )\n try:\n result = orchestrator.start_session(\n StartSessionCommand(\n user=current_user,\n environment_id=request.environment_id,\n source_kind=request.source_kind,\n source_input=request.source_input,\n )\n )\n except ValueError as exc:\n logger.explore(\n \"Session start rejected\",\n extra={\"user_id\": current_user.id, \"error\": str(exc)},\n )\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail == \"Environment not found\"\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n logger.reflect(\n \"Session started\", extra={\"session_id\": result.session.session_id}\n )\n return _serialize_session_summary(result.session)\n\n\n# [/DEF:start_session:Function]\n\n\n# [DEF:get_session_detail:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Return the full accessible dataset review session aggregate.\n@router.get(\n \"/sessions/{session_id}\",\n response_model=SessionSummary,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def get_session_detail(\n session_id: str,\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.get_session_detail\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n return _serialize_session_detail(session)\n\n\n# [/DEF:get_session_detail:Function]\n\n\n# [DEF:update_session:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Update resumable lifecycle status for an owned session.\n@router.patch(\n \"/sessions/{session_id}\",\n response_model=SessionSummary,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def update_session(\n session_id: str,\n request: UpdateSessionRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"update_session\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n session_record = cast(Any, session)\n session_record.status = request.status\n if request.status == SessionStatus.PAUSED:\n session_record.recommended_action = RecommendedAction.RESUME_SESSION\n elif request.status in {\n SessionStatus.ARCHIVED,\n SessionStatus.CANCELLED,\n SessionStatus.COMPLETED,\n }:\n session_record.active_task_id = None\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"session_status_updated\",\n event_summary=\"Dataset review session lifecycle updated\",\n event_details={\n \"status\": session_record.status.value,\n \"version\": session_record.version,\n },\n )\n return _serialize_session_summary(session)\n\n\n# [/DEF:update_session:Function]\n\n\n# [DEF:delete_session:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Archive or hard-delete a session owned by the current user.\n@router.delete(\n \"/sessions/{session_id}\",\n status_code=status.HTTP_204_NO_CONTENT,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def delete_session(\n session_id: str,\n hard_delete: bool = Query(False),\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"delete_session\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n if hard_delete:\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"session_deleted\",\n event_summary=\"Session hard-deleted\",\n event_details={\"hard_delete\": True},\n )\n repository.db.delete(session)\n repository.db.commit()\n return Response(status_code=status.HTTP_204_NO_CONTENT)\n session_record = cast(Any, session)\n session_record.status = SessionStatus.ARCHIVED\n session_record.active_task_id = None\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"session_archived\",\n event_summary=\"Session archived\",\n event_details={\"hard_delete\": False, \"version\": session_record.version},\n )\n return Response(status_code=status.HTTP_204_NO_CONTENT)\n\n\n# [/DEF:delete_session:Function]\n\n\n# [DEF:export_documentation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Export documentation output for the current session.\n@router.get(\n \"/sessions/{session_id}/exports/documentation\",\n response_model=ExportArtifactResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def export_documentation(\n session_id: str,\n format: ArtifactFormat = Query(ArtifactFormat.JSON),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"export_documentation\"):\n if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"Only json and markdown exports are supported\",\n )\n session = _get_owned_session_or_404(repository, session_id, current_user)\n payload = _build_documentation_export(session, format)\n return ExportArtifactResponse(\n artifact_id=f\"documentation-{session.session_id}-{format.value}\",\n session_id=session.session_id,\n artifact_type=\"documentation\",\n format=format.value,\n storage_ref=payload[\"storage_ref\"],\n created_by_user_id=current_user.id,\n content=payload[\"content\"],\n )\n\n\n# [/DEF:export_documentation:Function]\n\n\n# [DEF:export_validation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Export validation findings for the current session.\n@router.get(\n \"/sessions/{session_id}/exports/validation\",\n response_model=ExportArtifactResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def export_validation(\n session_id: str,\n format: ArtifactFormat = Query(ArtifactFormat.JSON),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"export_validation\"):\n if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"Only json and markdown exports are supported\",\n )\n session = _get_owned_session_or_404(repository, session_id, current_user)\n payload = _build_validation_export(session, format)\n return ExportArtifactResponse(\n artifact_id=f\"validation-{session.session_id}-{format.value}\",\n session_id=session.session_id,\n artifact_type=\"validation_report\",\n format=format.value,\n storage_ref=payload[\"storage_ref\"],\n created_by_user_id=current_user.id,\n content=payload[\"content\"],\n )\n\n\n# [/DEF:export_validation:Function]\n\n\n# [DEF:get_clarification_state:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Return the current clarification session summary and active question payload.\n@router.get(\n \"/sessions/{session_id}/clarification\",\n response_model=ClarificationStateResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def get_clarification_state(\n session_id: str,\n repository=Depends(_get_repository),\n clarification_engine=Depends(_get_clarification_engine),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"get_clarification_state\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n if not session.clarification_sessions:\n return _serialize_empty_clarification_state()\n cs = _get_latest_clarification_session_or_404(session)\n question = clarification_engine.build_question_payload(session)\n return _serialize_clarification_state(\n ClarificationStateResult(\n clarification_session=cs,\n current_question=question,\n session=session,\n changed_findings=[],\n )\n )\n\n\n# [/DEF:get_clarification_state:Function]\n\n\n# [DEF:resume_clarification:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Resume clarification mode on the highest-priority unresolved question.\n@router.post(\n \"/sessions/{session_id}/clarification/resume\",\n response_model=ClarificationStateResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def resume_clarification(\n session_id: str,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n clarification_engine=Depends(_get_clarification_engine),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"resume_clarification\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n cs = _get_latest_clarification_session_or_404(session)\n question = clarification_engine.build_question_payload(session)\n return _serialize_clarification_state(\n ClarificationStateResult(\n clarification_session=cs,\n current_question=question,\n session=session,\n changed_findings=[],\n )\n )\n\n\n# [/DEF:resume_clarification:Function]\n\n\n# [DEF:record_clarification_answer:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist one clarification answer before advancing the active pointer.\n@router.post(\n \"/sessions/{session_id}/clarification/answers\",\n response_model=ClarificationAnswerResultResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def record_clarification_answer(\n session_id: str,\n request: ClarificationAnswerRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n clarification_engine=Depends(_get_clarification_engine),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.record_clarification_answer\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n try:\n result = clarification_engine.record_answer(\n ClarificationAnswerCommand(\n session=session,\n question_id=request.question_id,\n answer_kind=request.answer_kind,\n answer_value=request.answer_value,\n user=current_user,\n )\n )\n except ValueError as exc:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)\n ) from exc\n return ClarificationAnswerResultResponse(\n clarification_state=_serialize_clarification_state(result),\n session=_serialize_session_summary(result.session),\n changed_findings=[\n ValidationFindingDto.model_validate(f, from_attributes=True)\n for f in result.changed_findings\n ],\n )\n\n\n# [/DEF:record_clarification_answer:Function]\n\n\n# [DEF:update_field_semantic:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Apply one field-level semantic candidate decision or manual override.\n@router.patch(\n \"/sessions/{session_id}/fields/{field_id}/semantic\",\n response_model=SemanticFieldEntryDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def update_field_semantic(\n session_id: str,\n field_id: str,\n request: FieldSemanticUpdateRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.update_field_semantic\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n _update_semantic_field_state(field, request, changed_by=\"user\")\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[field])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_updated\",\n event_summary=\"Semantic field decision persisted\",\n event_details={\"field_id\": field.field_id, \"version\": sr.version},\n )\n return _serialize_semantic_field(field)\n\n\n# [/DEF:update_field_semantic:Function]\n\n\n# [DEF:lock_field_semantic:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lock one semantic field against later automatic overwrite.\n@router.post(\n \"/sessions/{session_id}/fields/{field_id}/lock\",\n response_model=SemanticFieldEntryDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def lock_field_semantic(\n session_id: str,\n field_id: str,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.lock_field_semantic\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n field.is_locked = True\n field.last_changed_by = \"user\"\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[field])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_locked\",\n event_summary=\"Semantic field lock persisted\",\n event_details={\"field_id\": field.field_id, \"version\": sr.version},\n )\n return _serialize_semantic_field(field)\n\n\n# [/DEF:lock_field_semantic:Function]\n\n\n# [DEF:unlock_field_semantic:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Unlock one semantic field so later automated candidate application may replace it.\n@router.post(\n \"/sessions/{session_id}/fields/{field_id}/unlock\",\n response_model=SemanticFieldEntryDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def unlock_field_semantic(\n session_id: str,\n field_id: str,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.unlock_field_semantic\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n field.is_locked = False\n field.last_changed_by = \"user\"\n if field.provenance == FieldProvenance.MANUAL_OVERRIDE:\n field.provenance = FieldProvenance.UNRESOLVED\n field.needs_review = True\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[field])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_unlocked\",\n event_summary=\"Semantic field unlock persisted\",\n event_details={\"field_id\": field.field_id, \"version\": sr.version},\n )\n return _serialize_semantic_field(field)\n\n\n# [/DEF:unlock_field_semantic:Function]\n\n\n# [DEF:approve_batch_semantic_fields:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Approve multiple semantic candidate decisions in one batch.\n@router.post(\n \"/sessions/{session_id}/fields/semantic/approve-batch\",\n response_model=List[SemanticFieldEntryDto],\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def approve_batch_semantic_fields(\n session_id: str,\n request: BatchApproveSemanticRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.approve_batch_semantic_fields\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n updated = []\n for item in request.items:\n field = _get_owned_field_or_404(session, item.field_id)\n _update_semantic_field_state(\n field,\n FieldSemanticUpdateRequest(\n candidate_id=item.candidate_id, lock_field=item.lock_field\n ),\n changed_by=\"user\",\n )\n updated.append(field)\n sr = cast(Any, session)\n _commit_owned_session_mutation(\n repository, session, refresh_targets=list(updated)\n )\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_fields_batch_approved\",\n event_summary=\"Batch semantic approval persisted\",\n event_details={\"count\": len(updated), \"version\": sr.version},\n )\n return [_serialize_semantic_field(f) for f in updated]\n\n\n# [/DEF:approve_batch_semantic_fields:Function]\n\n\n# [DEF:list_execution_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Return the current mapping-review set for one accessible session.\n@router.get(\n \"/sessions/{session_id}/mappings\",\n response_model=MappingCollectionResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def list_execution_mappings(\n session_id: str,\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.list_execution_mappings\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n return MappingCollectionResponse(\n items=[_serialize_execution_mapping(m) for m in session.execution_mappings]\n )\n\n\n# [/DEF:list_execution_mappings:Function]\n\n\n# [DEF:update_execution_mapping:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist one owner-authorized execution-mapping effective value override.\n@router.patch(\n \"/sessions/{session_id}/mappings/{mapping_id}\",\n response_model=ExecutionMappingDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def update_execution_mapping(\n session_id: str,\n mapping_id: str,\n request: UpdateExecutionMappingRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.update_execution_mapping\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n mapping = _get_owned_mapping_or_404(session, mapping_id)\n if request.effective_value is None:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"effective_value is required\",\n )\n mapping.effective_value = request.effective_value\n mapping.mapping_method = MappingMethod(\n request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value\n )\n mapping.transformation_note = request.transformation_note\n mapping.approval_state = ApprovalState.APPROVED\n mapping.approved_by_user_id = current_user.id\n mapping.approved_at = datetime.utcnow()\n session.last_activity_at = datetime.utcnow()\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n if session.readiness_state in {\n ReadinessState.MAPPING_REVIEW_NEEDED,\n ReadinessState.COMPILED_PREVIEW_READY,\n ReadinessState.RUN_READY,\n ReadinessState.RUN_IN_PROGRESS,\n }:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n for preview in session.previews:\n if preview.preview_status == PreviewStatus.READY:\n preview.preview_status = PreviewStatus.STALE\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[mapping])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"execution_mapping_updated\",\n event_summary=\"Mapping override persisted\",\n event_details={\"mapping_id\": mapping.mapping_id, \"version\": sr.version},\n )\n return _serialize_execution_mapping(mapping)\n\n\n# [/DEF:update_execution_mapping:Function]\n\n\n# [DEF:approve_execution_mapping:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Explicitly approve a warning-sensitive mapping transformation.\n@router.post(\n \"/sessions/{session_id}/mappings/{mapping_id}/approve\",\n response_model=ExecutionMappingDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def approve_execution_mapping(\n session_id: str,\n mapping_id: str,\n request: ApproveMappingRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.approve_execution_mapping\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n mapping = _get_owned_mapping_or_404(session, mapping_id)\n mapping.approval_state = ApprovalState.APPROVED\n mapping.approved_by_user_id = current_user.id\n mapping.approved_at = datetime.utcnow()\n if request.approval_note:\n mapping.transformation_note = request.approval_note\n session.last_activity_at = datetime.utcnow()\n if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED:\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[mapping])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"execution_mapping_approved\",\n event_summary=\"Mapping approval persisted\",\n event_details={\"mapping_id\": mapping.mapping_id, \"version\": sr.version},\n )\n return _serialize_execution_mapping(mapping)\n\n\n# [/DEF:approve_execution_mapping:Function]\n\n\n# [DEF:approve_batch_execution_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Approve multiple warning-sensitive execution mappings in one batch.\n@router.post(\n \"/sessions/{session_id}/mappings/approve-batch\",\n response_model=List[ExecutionMappingDto],\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def approve_batch_execution_mappings(\n session_id: str,\n request: BatchApproveMappingRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.approve_batch_execution_mappings\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n updated = []\n for mid in list(dict.fromkeys(request.mapping_ids)):\n mapping = _get_owned_mapping_or_404(session, mid)\n mapping.approval_state = ApprovalState.APPROVED\n mapping.approved_by_user_id = current_user.id\n mapping.approved_at = datetime.utcnow()\n if request.approval_note:\n mapping.transformation_note = request.approval_note\n updated.append(mapping)\n session.last_activity_at = datetime.utcnow()\n if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED:\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n sr = cast(Any, session)\n _commit_owned_session_mutation(\n repository, session, refresh_targets=list(updated)\n )\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"execution_mappings_batch_approved\",\n event_summary=\"Batch mapping approval persisted\",\n event_details={\"count\": len(updated), \"version\": sr.version},\n )\n return [_serialize_execution_mapping(m) for m in updated]\n\n\n# [/DEF:approve_batch_execution_mappings:Function]\n\n\n# [DEF:trigger_preview_generation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Trigger Superset-side preview compilation for the current owned execution context.\n@router.post(\n \"/sessions/{session_id}/preview\",\n response_model=Union[CompiledPreviewDto, PreviewEnqueueResultResponse],\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def trigger_preview_generation(\n session_id: str,\n response: Response,\n orchestrator=Depends(_get_orchestrator),\n repository=Depends(_get_repository),\n session_version: int = Depends(_require_session_version_header),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.trigger_preview_generation\"):\n _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n try:\n result = orchestrator.prepare_launch_preview(\n PreparePreviewCommand(\n user=current_user,\n session_id=session_id,\n expected_version=session_version,\n )\n )\n except DatasetReviewSessionVersionConflictError as exc:\n raise _build_session_version_conflict_http_exception(exc) from exc\n except ValueError as exc:\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail in {\"Session not found\", \"Environment not found\"}\n else status.HTTP_409_CONFLICT\n if detail.startswith(\"Preview blocked:\")\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n if result.preview.preview_status == PreviewStatus.PENDING:\n response.status_code = status.HTTP_202_ACCEPTED\n return PreviewEnqueueResultResponse(\n session_id=result.session.session_id,\n session_version=int(getattr(result.session, \"version\", 0) or 0),\n preview_status=result.preview.preview_status.value,\n task_id=None,\n )\n response.status_code = status.HTTP_200_OK\n return _serialize_preview(\n result.preview,\n session_version_fallback=int(getattr(result.session, \"version\", 0) or 0),\n )\n\n\n# [/DEF:trigger_preview_generation:Function]\n\n\n# [DEF:launch_dataset:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Execute the current owned session launch handoff and return audited SQL Lab run context.\n@router.post(\n \"/sessions/{session_id}/launch\",\n response_model=LaunchDatasetResponse,\n status_code=status.HTTP_201_CREATED,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:execution:launch\", \"EXECUTE\")),\n ],\n)\nasync def launch_dataset(\n session_id: str,\n orchestrator=Depends(_get_orchestrator),\n repository=Depends(_get_repository),\n session_version: int = Depends(_require_session_version_header),\n config_manager=Depends(get_config_manager),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.launch_dataset\"):\n _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n try:\n result = orchestrator.launch_dataset(\n LaunchDatasetCommand(\n user=current_user,\n session_id=session_id,\n expected_version=session_version,\n )\n )\n except DatasetReviewSessionVersionConflictError as exc:\n raise _build_session_version_conflict_http_exception(exc) from exc\n except ValueError as exc:\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail in {\"Session not found\", \"Environment not found\"}\n else status.HTTP_409_CONFLICT\n if detail.startswith(\"Launch blocked:\")\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n environment = config_manager.get_environment(result.session.environment_id)\n env_url = getattr(environment, \"url\", \"\") if environment is not None else \"\"\n return LaunchDatasetResponse(\n run_context=_serialize_run_context(result.run_context),\n redirect_url=_build_sql_lab_redirect_url(\n environment_url=env_url,\n sql_lab_session_ref=result.run_context.sql_lab_session_ref,\n ),\n )\n\n\n# [/DEF:launch_dataset:Function]\n\n\n# [DEF:record_field_feedback:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist thumbs up/down feedback for AI-assisted semantic field content.\n@router.post(\n \"/sessions/{session_id}/fields/{field_id}/feedback\",\n response_model=FeedbackResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def record_field_feedback(\n session_id: str,\n field_id: str,\n request: FeedbackRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.record_field_feedback\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n field.user_feedback = request.feedback\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_feedback_recorded\",\n event_summary=\"Feedback persisted\",\n event_details={\n \"field_id\": field.field_id,\n \"feedback\": request.feedback,\n \"version\": sr.version,\n },\n )\n return FeedbackResponse(target_id=field.field_id, feedback=request.feedback)\n\n\n# [/DEF:record_field_feedback:Function]\n\n\n# [DEF:record_clarification_feedback:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist thumbs up/down feedback for clarification question/answer content.\n@router.post(\n \"/sessions/{session_id}/clarification/questions/{question_id}/feedback\",\n response_model=FeedbackResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def record_clarification_feedback(\n session_id: str,\n question_id: str,\n request: FeedbackRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.record_clarification_feedback\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n cs = _get_latest_clarification_session_or_404(session)\n question = next((q for q in cs.questions if q.question_id == question_id), None)\n if question is None:\n raise HTTPException(\n status_code=status.HTTP_404_NOT_FOUND,\n detail=\"Clarification question not found\",\n )\n if question.answer is None:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"Clarification answer not found\",\n )\n question.answer.user_feedback = request.feedback\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"clarification_feedback_recorded\",\n event_summary=\"Feedback persisted\",\n event_details={\n \"question_id\": question.question_id,\n \"feedback\": request.feedback,\n \"version\": sr.version,\n },\n )\n return FeedbackResponse(\n target_id=question.question_id, feedback=request.feedback\n )\n\n\n# [/DEF:record_clarification_feedback:Function]\n\n\n# [/DEF:DatasetReviewRoutes:Module]\n" + "body": "# [DEF:DatasetReviewRoutes:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist thumbs up/down feedback for clarification question/answer content.\n\n\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import Any, List, Optional, Union, cast\n\nfrom fastapi import APIRouter, Depends, HTTPException, Query, Response, status\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.dependencies import get_config_manager, get_current_user, has_permission\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n ApprovalState,\n ArtifactFormat,\n FieldProvenance,\n MappingMethod,\n PreviewStatus,\n ReadinessState,\n RecommendedAction,\n SessionStatus,\n)\nfrom src.schemas.dataset_review import (\n ClarificationAnswerDto,\n CompiledPreviewDto,\n ExecutionMappingDto,\n SemanticFieldEntryDto,\n SessionSummary,\n ValidationFindingDto,\n)\nfrom src.services.dataset_review.clarification_engine import (\n ClarificationAnswerCommand,\n ClarificationStateResult,\n)\nfrom src.services.dataset_review.orchestrator import (\n LaunchDatasetCommand,\n PreparePreviewCommand,\n StartSessionCommand,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionVersionConflictError,\n)\nfrom src.api.routes.dataset_review_pkg._dependencies import (\n BatchApproveMappingRequest,\n BatchApproveSemanticRequest,\n ClarificationAnswerRequest,\n ClarificationAnswerResultResponse,\n ClarificationStateResponse,\n ExportArtifactResponse,\n FeedbackRequest,\n FeedbackResponse,\n FieldSemanticUpdateRequest,\n LaunchDatasetResponse,\n MappingCollectionResponse,\n PreviewEnqueueResultResponse,\n SessionCollectionResponse,\n StartSessionRequest,\n UpdateExecutionMappingRequest,\n UpdateSessionRequest,\n _build_documentation_export,\n _build_sql_lab_redirect_url,\n _build_validation_export,\n _commit_owned_session_mutation,\n _get_clarification_engine,\n _get_latest_clarification_session_or_404,\n _get_owned_field_or_404,\n _get_owned_mapping_or_404,\n _get_owned_session_or_404,\n _get_orchestrator,\n _get_repository,\n _prepare_owned_session_mutation,\n _record_session_event,\n _require_auto_review_flag,\n _require_clarification_flag,\n _require_execution_flag,\n _require_session_version_header,\n _serialize_clarification_state,\n _serialize_empty_clarification_state,\n _serialize_execution_mapping,\n _serialize_preview,\n _serialize_run_context,\n _serialize_semantic_field,\n _serialize_session_detail,\n _serialize_session_summary,\n _update_semantic_field_state,\n _build_session_version_conflict_http_exception,\n)\n\nlog = MarkerLogger(\"DatasetReviewRoutes\")\n\nrouter = APIRouter(prefix=\"/api/dataset-orchestration\", tags=[\"Dataset Orchestration\"])\n\n\n# [DEF:list_sessions:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: List resumable dataset review sessions for the current user.\n@router.get(\n \"/sessions\",\n response_model=SessionCollectionResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def list_sessions(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.list_sessions\"):\n log.reason(\n \"Listing dataset review sessions\",\n payload={\"user_id\": current_user.id, \"page\": page, \"page_size\": page_size},\n )\n sessions = repository.list_sessions_for_user(current_user.id)\n start = (page - 1) * page_size\n end = start + page_size\n items = [_serialize_session_summary(s) for s in sessions[start:end]]\n log.reflect(\n \"Session page assembled\",\n payload={\n \"user_id\": current_user.id,\n \"returned\": len(items),\n \"total\": len(sessions),\n },\n )\n return SessionCollectionResponse(\n items=items,\n total=len(sessions),\n page=page,\n page_size=page_size,\n has_next=end < len(sessions),\n )\n\n\n# [/DEF:list_sessions:Function]\n\n\n# [DEF:start_session:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Start a new dataset review session from a Superset link or dataset selection.\n@router.post(\n \"/sessions\",\n response_model=SessionSummary,\n status_code=status.HTTP_201_CREATED,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def start_session(\n request: StartSessionRequest,\n orchestrator=Depends(_get_orchestrator),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"start_session\"):\n log.reason(\n \"Starting dataset review session\",\n payload={\n \"user_id\": current_user.id,\n \"environment_id\": request.environment_id,\n },\n )\n try:\n result = orchestrator.start_session(\n StartSessionCommand(\n user=current_user,\n environment_id=request.environment_id,\n source_kind=request.source_kind,\n source_input=request.source_input,\n )\n )\n except ValueError as exc:\n log.explore(\"Session start rejected\", error=\"Session start rejected by orchestrator\", payload={\"user_id\": current_user.id, \"error\": str(exc)})\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail == \"Environment not found\"\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n log.reflect(\n \"Session started\", payload={\"session_id\": result.session.session_id}\n )\n return _serialize_session_summary(result.session)\n\n\n# [/DEF:start_session:Function]\n\n\n# [DEF:get_session_detail:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Return the full accessible dataset review session aggregate.\n@router.get(\n \"/sessions/{session_id}\",\n response_model=SessionSummary,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def get_session_detail(\n session_id: str,\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.get_session_detail\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n return _serialize_session_detail(session)\n\n\n# [/DEF:get_session_detail:Function]\n\n\n# [DEF:update_session:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Update resumable lifecycle status for an owned session.\n@router.patch(\n \"/sessions/{session_id}\",\n response_model=SessionSummary,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def update_session(\n session_id: str,\n request: UpdateSessionRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"update_session\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n session_record = cast(Any, session)\n session_record.status = request.status\n if request.status == SessionStatus.PAUSED:\n session_record.recommended_action = RecommendedAction.RESUME_SESSION\n elif request.status in {\n SessionStatus.ARCHIVED,\n SessionStatus.CANCELLED,\n SessionStatus.COMPLETED,\n }:\n session_record.active_task_id = None\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"session_status_updated\",\n event_summary=\"Dataset review session lifecycle updated\",\n event_details={\n \"status\": session_record.status.value,\n \"version\": session_record.version,\n },\n )\n return _serialize_session_summary(session)\n\n\n# [/DEF:update_session:Function]\n\n\n# [DEF:delete_session:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Archive or hard-delete a session owned by the current user.\n@router.delete(\n \"/sessions/{session_id}\",\n status_code=status.HTTP_204_NO_CONTENT,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def delete_session(\n session_id: str,\n hard_delete: bool = Query(False),\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"delete_session\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n if hard_delete:\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"session_deleted\",\n event_summary=\"Session hard-deleted\",\n event_details={\"hard_delete\": True},\n )\n repository.db.delete(session)\n repository.db.commit()\n return Response(status_code=status.HTTP_204_NO_CONTENT)\n session_record = cast(Any, session)\n session_record.status = SessionStatus.ARCHIVED\n session_record.active_task_id = None\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"session_archived\",\n event_summary=\"Session archived\",\n event_details={\"hard_delete\": False, \"version\": session_record.version},\n )\n return Response(status_code=status.HTTP_204_NO_CONTENT)\n\n\n# [/DEF:delete_session:Function]\n\n\n# [DEF:export_documentation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Export documentation output for the current session.\n@router.get(\n \"/sessions/{session_id}/exports/documentation\",\n response_model=ExportArtifactResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def export_documentation(\n session_id: str,\n format: ArtifactFormat = Query(ArtifactFormat.JSON),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"export_documentation\"):\n if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"Only json and markdown exports are supported\",\n )\n session = _get_owned_session_or_404(repository, session_id, current_user)\n payload = _build_documentation_export(session, format)\n return ExportArtifactResponse(\n artifact_id=f\"documentation-{session.session_id}-{format.value}\",\n session_id=session.session_id,\n artifact_type=\"documentation\",\n format=format.value,\n storage_ref=payload[\"storage_ref\"],\n created_by_user_id=current_user.id,\n content=payload[\"content\"],\n )\n\n\n# [/DEF:export_documentation:Function]\n\n\n# [DEF:export_validation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Export validation findings for the current session.\n@router.get(\n \"/sessions/{session_id}/exports/validation\",\n response_model=ExportArtifactResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def export_validation(\n session_id: str,\n format: ArtifactFormat = Query(ArtifactFormat.JSON),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"export_validation\"):\n if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"Only json and markdown exports are supported\",\n )\n session = _get_owned_session_or_404(repository, session_id, current_user)\n payload = _build_validation_export(session, format)\n return ExportArtifactResponse(\n artifact_id=f\"validation-{session.session_id}-{format.value}\",\n session_id=session.session_id,\n artifact_type=\"validation_report\",\n format=format.value,\n storage_ref=payload[\"storage_ref\"],\n created_by_user_id=current_user.id,\n content=payload[\"content\"],\n )\n\n\n# [/DEF:export_validation:Function]\n\n\n# [DEF:get_clarification_state:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Return the current clarification session summary and active question payload.\n@router.get(\n \"/sessions/{session_id}/clarification\",\n response_model=ClarificationStateResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def get_clarification_state(\n session_id: str,\n repository=Depends(_get_repository),\n clarification_engine=Depends(_get_clarification_engine),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"get_clarification_state\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n if not session.clarification_sessions:\n return _serialize_empty_clarification_state()\n cs = _get_latest_clarification_session_or_404(session)\n question = clarification_engine.build_question_payload(session)\n return _serialize_clarification_state(\n ClarificationStateResult(\n clarification_session=cs,\n current_question=question,\n session=session,\n changed_findings=[],\n )\n )\n\n\n# [/DEF:get_clarification_state:Function]\n\n\n# [DEF:resume_clarification:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Resume clarification mode on the highest-priority unresolved question.\n@router.post(\n \"/sessions/{session_id}/clarification/resume\",\n response_model=ClarificationStateResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def resume_clarification(\n session_id: str,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n clarification_engine=Depends(_get_clarification_engine),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"resume_clarification\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n cs = _get_latest_clarification_session_or_404(session)\n question = clarification_engine.build_question_payload(session)\n return _serialize_clarification_state(\n ClarificationStateResult(\n clarification_session=cs,\n current_question=question,\n session=session,\n changed_findings=[],\n )\n )\n\n\n# [/DEF:resume_clarification:Function]\n\n\n# [DEF:record_clarification_answer:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist one clarification answer before advancing the active pointer.\n@router.post(\n \"/sessions/{session_id}/clarification/answers\",\n response_model=ClarificationAnswerResultResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def record_clarification_answer(\n session_id: str,\n request: ClarificationAnswerRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n clarification_engine=Depends(_get_clarification_engine),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.record_clarification_answer\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n try:\n result = clarification_engine.record_answer(\n ClarificationAnswerCommand(\n session=session,\n question_id=request.question_id,\n answer_kind=request.answer_kind,\n answer_value=request.answer_value,\n user=current_user,\n )\n )\n except ValueError as exc:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)\n ) from exc\n return ClarificationAnswerResultResponse(\n clarification_state=_serialize_clarification_state(result),\n session=_serialize_session_summary(result.session),\n changed_findings=[\n ValidationFindingDto.model_validate(f, from_attributes=True)\n for f in result.changed_findings\n ],\n )\n\n\n# [/DEF:record_clarification_answer:Function]\n\n\n# [DEF:update_field_semantic:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Apply one field-level semantic candidate decision or manual override.\n@router.patch(\n \"/sessions/{session_id}/fields/{field_id}/semantic\",\n response_model=SemanticFieldEntryDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def update_field_semantic(\n session_id: str,\n field_id: str,\n request: FieldSemanticUpdateRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.update_field_semantic\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n _update_semantic_field_state(field, request, changed_by=\"user\")\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[field])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_updated\",\n event_summary=\"Semantic field decision persisted\",\n event_details={\"field_id\": field.field_id, \"version\": sr.version},\n )\n return _serialize_semantic_field(field)\n\n\n# [/DEF:update_field_semantic:Function]\n\n\n# [DEF:lock_field_semantic:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Lock one semantic field against later automatic overwrite.\n@router.post(\n \"/sessions/{session_id}/fields/{field_id}/lock\",\n response_model=SemanticFieldEntryDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def lock_field_semantic(\n session_id: str,\n field_id: str,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.lock_field_semantic\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n field.is_locked = True\n field.last_changed_by = \"user\"\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[field])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_locked\",\n event_summary=\"Semantic field lock persisted\",\n event_details={\"field_id\": field.field_id, \"version\": sr.version},\n )\n return _serialize_semantic_field(field)\n\n\n# [/DEF:lock_field_semantic:Function]\n\n\n# [DEF:unlock_field_semantic:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Unlock one semantic field so later automated candidate application may replace it.\n@router.post(\n \"/sessions/{session_id}/fields/{field_id}/unlock\",\n response_model=SemanticFieldEntryDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def unlock_field_semantic(\n session_id: str,\n field_id: str,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.unlock_field_semantic\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n field.is_locked = False\n field.last_changed_by = \"user\"\n if field.provenance == FieldProvenance.MANUAL_OVERRIDE:\n field.provenance = FieldProvenance.UNRESOLVED\n field.needs_review = True\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[field])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_unlocked\",\n event_summary=\"Semantic field unlock persisted\",\n event_details={\"field_id\": field.field_id, \"version\": sr.version},\n )\n return _serialize_semantic_field(field)\n\n\n# [/DEF:unlock_field_semantic:Function]\n\n\n# [DEF:approve_batch_semantic_fields:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Approve multiple semantic candidate decisions in one batch.\n@router.post(\n \"/sessions/{session_id}/fields/semantic/approve-batch\",\n response_model=List[SemanticFieldEntryDto],\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def approve_batch_semantic_fields(\n session_id: str,\n request: BatchApproveSemanticRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.approve_batch_semantic_fields\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n updated = []\n for item in request.items:\n field = _get_owned_field_or_404(session, item.field_id)\n _update_semantic_field_state(\n field,\n FieldSemanticUpdateRequest(\n candidate_id=item.candidate_id, lock_field=item.lock_field\n ),\n changed_by=\"user\",\n )\n updated.append(field)\n sr = cast(Any, session)\n _commit_owned_session_mutation(\n repository, session, refresh_targets=list(updated)\n )\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_fields_batch_approved\",\n event_summary=\"Batch semantic approval persisted\",\n event_details={\"count\": len(updated), \"version\": sr.version},\n )\n return [_serialize_semantic_field(f) for f in updated]\n\n\n# [/DEF:approve_batch_semantic_fields:Function]\n\n\n# [DEF:list_execution_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Return the current mapping-review set for one accessible session.\n@router.get(\n \"/sessions/{session_id}/mappings\",\n response_model=MappingCollectionResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def list_execution_mappings(\n session_id: str,\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.list_execution_mappings\"):\n session = _get_owned_session_or_404(repository, session_id, current_user)\n return MappingCollectionResponse(\n items=[_serialize_execution_mapping(m) for m in session.execution_mappings]\n )\n\n\n# [/DEF:list_execution_mappings:Function]\n\n\n# [DEF:update_execution_mapping:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist one owner-authorized execution-mapping effective value override.\n@router.patch(\n \"/sessions/{session_id}/mappings/{mapping_id}\",\n response_model=ExecutionMappingDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def update_execution_mapping(\n session_id: str,\n mapping_id: str,\n request: UpdateExecutionMappingRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.update_execution_mapping\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n mapping = _get_owned_mapping_or_404(session, mapping_id)\n if request.effective_value is None:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"effective_value is required\",\n )\n mapping.effective_value = request.effective_value\n mapping.mapping_method = MappingMethod(\n request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value\n )\n mapping.transformation_note = request.transformation_note\n mapping.approval_state = ApprovalState.APPROVED\n mapping.approved_by_user_id = current_user.id\n mapping.approved_at = datetime.utcnow()\n session.last_activity_at = datetime.utcnow()\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n if session.readiness_state in {\n ReadinessState.MAPPING_REVIEW_NEEDED,\n ReadinessState.COMPILED_PREVIEW_READY,\n ReadinessState.RUN_READY,\n ReadinessState.RUN_IN_PROGRESS,\n }:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n for preview in session.previews:\n if preview.preview_status == PreviewStatus.READY:\n preview.preview_status = PreviewStatus.STALE\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[mapping])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"execution_mapping_updated\",\n event_summary=\"Mapping override persisted\",\n event_details={\"mapping_id\": mapping.mapping_id, \"version\": sr.version},\n )\n return _serialize_execution_mapping(mapping)\n\n\n# [/DEF:update_execution_mapping:Function]\n\n\n# [DEF:approve_execution_mapping:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Explicitly approve a warning-sensitive mapping transformation.\n@router.post(\n \"/sessions/{session_id}/mappings/{mapping_id}/approve\",\n response_model=ExecutionMappingDto,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def approve_execution_mapping(\n session_id: str,\n mapping_id: str,\n request: ApproveMappingRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.approve_execution_mapping\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n mapping = _get_owned_mapping_or_404(session, mapping_id)\n mapping.approval_state = ApprovalState.APPROVED\n mapping.approved_by_user_id = current_user.id\n mapping.approved_at = datetime.utcnow()\n if request.approval_note:\n mapping.transformation_note = request.approval_note\n session.last_activity_at = datetime.utcnow()\n if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED:\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session, refresh_targets=[mapping])\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"execution_mapping_approved\",\n event_summary=\"Mapping approval persisted\",\n event_details={\"mapping_id\": mapping.mapping_id, \"version\": sr.version},\n )\n return _serialize_execution_mapping(mapping)\n\n\n# [/DEF:approve_execution_mapping:Function]\n\n\n# [DEF:approve_batch_execution_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Approve multiple warning-sensitive execution mappings in one batch.\n@router.post(\n \"/sessions/{session_id}/mappings/approve-batch\",\n response_model=List[ExecutionMappingDto],\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def approve_batch_execution_mappings(\n session_id: str,\n request: BatchApproveMappingRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.approve_batch_execution_mappings\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n updated = []\n for mid in list(dict.fromkeys(request.mapping_ids)):\n mapping = _get_owned_mapping_or_404(session, mid)\n mapping.approval_state = ApprovalState.APPROVED\n mapping.approved_by_user_id = current_user.id\n mapping.approved_at = datetime.utcnow()\n if request.approval_note:\n mapping.transformation_note = request.approval_note\n updated.append(mapping)\n session.last_activity_at = datetime.utcnow()\n if session.readiness_state == ReadinessState.MAPPING_REVIEW_NEEDED:\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n sr = cast(Any, session)\n _commit_owned_session_mutation(\n repository, session, refresh_targets=list(updated)\n )\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"execution_mappings_batch_approved\",\n event_summary=\"Batch mapping approval persisted\",\n event_details={\"count\": len(updated), \"version\": sr.version},\n )\n return [_serialize_execution_mapping(m) for m in updated]\n\n\n# [/DEF:approve_batch_execution_mappings:Function]\n\n\n# [DEF:trigger_preview_generation:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Trigger Superset-side preview compilation for the current owned execution context.\n@router.post(\n \"/sessions/{session_id}/preview\",\n response_model=Union[CompiledPreviewDto, PreviewEnqueueResultResponse],\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def trigger_preview_generation(\n session_id: str,\n response: Response,\n orchestrator=Depends(_get_orchestrator),\n repository=Depends(_get_repository),\n session_version: int = Depends(_require_session_version_header),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.trigger_preview_generation\"):\n _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n try:\n result = orchestrator.prepare_launch_preview(\n PreparePreviewCommand(\n user=current_user,\n session_id=session_id,\n expected_version=session_version,\n )\n )\n except DatasetReviewSessionVersionConflictError as exc:\n raise _build_session_version_conflict_http_exception(exc) from exc\n except ValueError as exc:\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail in {\"Session not found\", \"Environment not found\"}\n else status.HTTP_409_CONFLICT\n if detail.startswith(\"Preview blocked:\")\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n if result.preview.preview_status == PreviewStatus.PENDING:\n response.status_code = status.HTTP_202_ACCEPTED\n return PreviewEnqueueResultResponse(\n session_id=result.session.session_id,\n session_version=int(getattr(result.session, \"version\", 0) or 0),\n preview_status=result.preview.preview_status.value,\n task_id=None,\n )\n response.status_code = status.HTTP_200_OK\n return _serialize_preview(\n result.preview,\n session_version_fallback=int(getattr(result.session, \"version\", 0) or 0),\n )\n\n\n# [/DEF:trigger_preview_generation:Function]\n\n\n# [DEF:launch_dataset:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Execute the current owned session launch handoff and return audited SQL Lab run context.\n@router.post(\n \"/sessions/{session_id}/launch\",\n response_model=LaunchDatasetResponse,\n status_code=status.HTTP_201_CREATED,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_execution_flag),\n Depends(has_permission(\"dataset:execution:launch\", \"EXECUTE\")),\n ],\n)\nasync def launch_dataset(\n session_id: str,\n orchestrator=Depends(_get_orchestrator),\n repository=Depends(_get_repository),\n session_version: int = Depends(_require_session_version_header),\n config_manager=Depends(get_config_manager),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.launch_dataset\"):\n _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n try:\n result = orchestrator.launch_dataset(\n LaunchDatasetCommand(\n user=current_user,\n session_id=session_id,\n expected_version=session_version,\n )\n )\n except DatasetReviewSessionVersionConflictError as exc:\n raise _build_session_version_conflict_http_exception(exc) from exc\n except ValueError as exc:\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail in {\"Session not found\", \"Environment not found\"}\n else status.HTTP_409_CONFLICT\n if detail.startswith(\"Launch blocked:\")\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n environment = config_manager.get_environment(result.session.environment_id)\n env_url = getattr(environment, \"url\", \"\") if environment is not None else \"\"\n return LaunchDatasetResponse(\n run_context=_serialize_run_context(result.run_context),\n redirect_url=_build_sql_lab_redirect_url(\n environment_url=env_url,\n sql_lab_session_ref=result.run_context.sql_lab_session_ref,\n ),\n )\n\n\n# [/DEF:launch_dataset:Function]\n\n\n# [DEF:record_field_feedback:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist thumbs up/down feedback for AI-assisted semantic field content.\n@router.post(\n \"/sessions/{session_id}/fields/{field_id}/feedback\",\n response_model=FeedbackResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def record_field_feedback(\n session_id: str,\n field_id: str,\n request: FeedbackRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.record_field_feedback\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n field = _get_owned_field_or_404(session, field_id)\n field.user_feedback = request.feedback\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"semantic_field_feedback_recorded\",\n event_summary=\"Feedback persisted\",\n event_details={\n \"field_id\": field.field_id,\n \"feedback\": request.feedback,\n \"version\": sr.version,\n },\n )\n return FeedbackResponse(target_id=field.field_id, feedback=request.feedback)\n\n\n# [/DEF:record_field_feedback:Function]\n\n\n# [DEF:record_clarification_feedback:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist thumbs up/down feedback for clarification question/answer content.\n@router.post(\n \"/sessions/{session_id}/clarification/questions/{question_id}/feedback\",\n response_model=FeedbackResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(_require_clarification_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def record_clarification_feedback(\n session_id: str,\n question_id: str,\n request: FeedbackRequest,\n session_version: int = Depends(_require_session_version_header),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.record_clarification_feedback\"):\n session = _prepare_owned_session_mutation(\n repository, session_id, current_user, session_version\n )\n cs = _get_latest_clarification_session_or_404(session)\n question = next((q for q in cs.questions if q.question_id == question_id), None)\n if question is None:\n raise HTTPException(\n status_code=status.HTTP_404_NOT_FOUND,\n detail=\"Clarification question not found\",\n )\n if question.answer is None:\n raise HTTPException(\n status_code=status.HTTP_400_BAD_REQUEST,\n detail=\"Clarification answer not found\",\n )\n question.answer.user_feedback = request.feedback\n sr = cast(Any, session)\n _commit_owned_session_mutation(repository, session)\n _record_session_event(\n repository,\n session,\n current_user,\n event_type=\"clarification_feedback_recorded\",\n event_summary=\"Feedback persisted\",\n event_details={\n \"question_id\": question.question_id,\n \"feedback\": request.feedback,\n \"version\": sr.version,\n },\n )\n return FeedbackResponse(\n target_id=question.question_id, feedback=request.feedback\n )\n\n\n# [/DEF:record_clarification_feedback:Function]\n\n\n# [/DEF:DatasetReviewRoutes:Module]\n" }, { "contract_id": "list_sessions", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_routes.py", - "start_line": 95, - "end_line": 138, + "start_line": 98, + "end_line": 141, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -16439,13 +16439,13 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:list_sessions:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: List resumable dataset review sessions for the current user.\n@router.get(\n \"/sessions\",\n response_model=SessionCollectionResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def list_sessions(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.list_sessions\"):\n logger.reason(\n \"Listing dataset review sessions\",\n extra={\"user_id\": current_user.id, \"page\": page, \"page_size\": page_size},\n )\n sessions = repository.list_sessions_for_user(current_user.id)\n start = (page - 1) * page_size\n end = start + page_size\n items = [_serialize_session_summary(s) for s in sessions[start:end]]\n logger.reflect(\n \"Session page assembled\",\n extra={\n \"user_id\": current_user.id,\n \"returned\": len(items),\n \"total\": len(sessions),\n },\n )\n return SessionCollectionResponse(\n items=items,\n total=len(sessions),\n page=page,\n page_size=page_size,\n has_next=end < len(sessions),\n )\n\n\n# [/DEF:list_sessions:Function]\n" + "body": "# [DEF:list_sessions:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: List resumable dataset review sessions for the current user.\n@router.get(\n \"/sessions\",\n response_model=SessionCollectionResponse,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"READ\")),\n ],\n)\nasync def list_sessions(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n repository=Depends(_get_repository),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"dataset_review.list_sessions\"):\n log.reason(\n \"Listing dataset review sessions\",\n payload={\"user_id\": current_user.id, \"page\": page, \"page_size\": page_size},\n )\n sessions = repository.list_sessions_for_user(current_user.id)\n start = (page - 1) * page_size\n end = start + page_size\n items = [_serialize_session_summary(s) for s in sessions[start:end]]\n log.reflect(\n \"Session page assembled\",\n payload={\n \"user_id\": current_user.id,\n \"returned\": len(items),\n \"total\": len(sessions),\n },\n )\n return SessionCollectionResponse(\n items=items,\n total=len(sessions),\n page=page,\n page_size=page_size,\n has_next=end < len(sessions),\n )\n\n\n# [/DEF:list_sessions:Function]\n" }, { "contract_id": "start_session", "contract_type": "Function", "file_path": "backend/src/api/routes/dataset_review_pkg/_routes.py", - "start_line": 141, + "start_line": 144, "end_line": 193, "tier": "TIER_2", "complexity": 4, @@ -16493,7 +16493,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:start_session:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Start a new dataset review session from a Superset link or dataset selection.\n@router.post(\n \"/sessions\",\n response_model=SessionSummary,\n status_code=status.HTTP_201_CREATED,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def start_session(\n request: StartSessionRequest,\n orchestrator=Depends(_get_orchestrator),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"start_session\"):\n logger.reason(\n \"Starting dataset review session\",\n extra={\n \"user_id\": current_user.id,\n \"environment_id\": request.environment_id,\n },\n )\n try:\n result = orchestrator.start_session(\n StartSessionCommand(\n user=current_user,\n environment_id=request.environment_id,\n source_kind=request.source_kind,\n source_input=request.source_input,\n )\n )\n except ValueError as exc:\n logger.explore(\n \"Session start rejected\",\n extra={\"user_id\": current_user.id, \"error\": str(exc)},\n )\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail == \"Environment not found\"\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n logger.reflect(\n \"Session started\", extra={\"session_id\": result.session.session_id}\n )\n return _serialize_session_summary(result.session)\n\n\n# [/DEF:start_session:Function]\n" + "body": "# [DEF:start_session:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Start a new dataset review session from a Superset link or dataset selection.\n@router.post(\n \"/sessions\",\n response_model=SessionSummary,\n status_code=status.HTTP_201_CREATED,\n dependencies=[\n Depends(_require_auto_review_flag),\n Depends(has_permission(\"dataset:session\", \"MANAGE\")),\n ],\n)\nasync def start_session(\n request: StartSessionRequest,\n orchestrator=Depends(_get_orchestrator),\n current_user: User = Depends(get_current_user),\n):\n with belief_scope(\"start_session\"):\n log.reason(\n \"Starting dataset review session\",\n payload={\n \"user_id\": current_user.id,\n \"environment_id\": request.environment_id,\n },\n )\n try:\n result = orchestrator.start_session(\n StartSessionCommand(\n user=current_user,\n environment_id=request.environment_id,\n source_kind=request.source_kind,\n source_input=request.source_input,\n )\n )\n except ValueError as exc:\n log.explore(\"Session start rejected\", error=\"Session start rejected by orchestrator\", payload={\"user_id\": current_user.id, \"error\": str(exc)})\n detail = str(exc)\n sc = (\n status.HTTP_404_NOT_FOUND\n if detail == \"Environment not found\"\n else status.HTTP_400_BAD_REQUEST\n )\n raise HTTPException(status_code=sc, detail=detail) from exc\n log.reflect(\n \"Session started\", payload={\"session_id\": result.session.session_id}\n )\n return _serialize_session_summary(result.session)\n\n\n# [/DEF:start_session:Function]\n" }, { "contract_id": "get_session_detail", @@ -18822,7 +18822,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/git/_helpers.py", "start_line": 1, - "end_line": 345, + "end_line": 343, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -18924,14 +18924,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitHelpers:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: Shared helper functions for Git route modules.\n# @LAYER: API\n# @SEMANTICS: git, helpers, resolution, identity\n# @RELATION USES -> [GitDeps]\n# @RELATION USES -> [SupersetClient]\n# @RELATION USES -> [UserDashboardPreference]\n\nimport os\nfrom typing import Optional\n\nfrom fastapi import HTTPException\nfrom sqlalchemy.orm import Session\n\nfrom src.core.logger import logger\nfrom src.core.superset_client import SupersetClient\nfrom src.dependencies import get_config_manager\nfrom src.models.auth import User\nfrom src.models.git import GitServerConfig\nfrom src.models.profile import UserDashboardPreference\n\nfrom ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH\n\n\n# [DEF:_build_no_repo_status_payload:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build a consistent status payload for dashboards without initialized repositories.\n# @POST: Returns a stable payload compatible with frontend repository status parsing.\ndef _build_no_repo_status_payload() -> dict:\n return {\n \"is_dirty\": False,\n \"untracked_files\": [],\n \"modified_files\": [],\n \"staged_files\": [],\n \"current_branch\": None,\n \"upstream_branch\": None,\n \"has_upstream\": False,\n \"ahead_count\": 0,\n \"behind_count\": 0,\n \"is_diverged\": False,\n \"sync_state\": \"NO_REPO\",\n \"sync_status\": \"NO_REPO\",\n \"has_repo\": False,\n }\n# [/DEF:_build_no_repo_status_payload:Function]\n\n\n# [DEF:_handle_unexpected_git_route_error:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses.\n# @PRE: `error` is a non-HTTPException instance.\n# @POST: Raises HTTPException(500) with route-specific context.\ndef _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None:\n logger.error(f\"[{route_name}][Coherence:Failed] {error}\")\n raise HTTPException(status_code=500, detail=f\"{route_name} failed: {str(error)}\")\n# [/DEF:_handle_unexpected_git_route_error:Function]\n\n\n# [DEF:_resolve_repository_status:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics.\n# @PRE: `dashboard_id` is a valid integer.\n# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent.\ndef _resolve_repository_status(dashboard_id: int) -> dict:\n git_service = get_git_service()\n repo_path = git_service._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n logger.debug(\n f\"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n\n try:\n return git_service.get_status(dashboard_id)\n except HTTPException as e:\n if e.status_code == 404:\n logger.debug(\n f\"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n raise\n# [/DEF:_resolve_repository_status:Function]\n\n\n# [DEF:_get_git_config_or_404:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve GitServerConfig by id or raise 404.\ndef _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig:\n config = db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first()\n if not config:\n raise HTTPException(status_code=404, detail=\"Git configuration not found\")\n return config\n# [/DEF:_get_git_config_or_404:Function]\n\n\n# [DEF:_find_dashboard_id_by_slug:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard numeric ID by slug in a specific environment.\ndef _find_dashboard_id_by_slug(\n client: SupersetClient,\n dashboard_slug: str,\n) -> Optional[int]:\n query_variants = [\n {\n \"filters\": [{\"col\": \"slug\", \"opr\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n {\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n ]\n\n for query in query_variants:\n try:\n _count, dashboards = client.get_dashboards_page(query=query)\n if dashboards:\n resolved_id = dashboards[0].get(\"id\")\n if resolved_id is not None:\n return int(resolved_id)\n except Exception:\n continue\n return None\n# [/DEF:_find_dashboard_id_by_slug:Function]\n\n\n# [DEF:_resolve_dashboard_id_from_ref:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard ID from slug-or-id reference for Git routes.\ndef _resolve_dashboard_id_from_ref(\n dashboard_ref: str,\n config_manager,\n env_id: Optional[str] = None,\n) -> int:\n normalized_ref = str(dashboard_ref or \"\").strip()\n if not normalized_ref:\n raise HTTPException(status_code=400, detail=\"dashboard_ref is required\")\n\n if normalized_ref.isdigit():\n return int(normalized_ref)\n\n if not env_id:\n raise HTTPException(\n status_code=400,\n detail=\"env_id is required for slug-based Git operations\",\n )\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n dashboard_id = _find_dashboard_id_by_slug(SupersetClient(env), normalized_ref)\n if dashboard_id is None:\n raise HTTPException(\n status_code=404, detail=f\"Dashboard slug '{normalized_ref}' not found\"\n )\n return dashboard_id\n# [/DEF:_resolve_dashboard_id_from_ref:Function]\n\n\n# [DEF:_find_dashboard_id_by_slug_async:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes.\nasync def _find_dashboard_id_by_slug_async(\n client: \"AsyncSupersetClient\",\n dashboard_slug: str,\n) -> Optional[int]:\n query_variants = [\n {\n \"filters\": [{\"col\": \"slug\", \"opr\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n {\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n ]\n\n for query in query_variants:\n try:\n _count, dashboards = await client.get_dashboards_page_async(query=query)\n if dashboards:\n resolved_id = dashboards[0].get(\"id\")\n if resolved_id is not None:\n return int(resolved_id)\n except Exception:\n continue\n return None\n# [/DEF:_find_dashboard_id_by_slug_async:Function]\n\n\n# [DEF:_resolve_dashboard_id_from_ref_async:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes.\nasync def _resolve_dashboard_id_from_ref_async(\n dashboard_ref: str,\n config_manager,\n env_id: Optional[str] = None,\n) -> int:\n normalized_ref = str(dashboard_ref or \"\").strip()\n if not normalized_ref:\n raise HTTPException(status_code=400, detail=\"dashboard_ref is required\")\n\n if normalized_ref.isdigit():\n return int(normalized_ref)\n\n if not env_id:\n raise HTTPException(\n status_code=400,\n detail=\"env_id is required for slug-based Git operations\",\n )\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n from src.core.async_superset_client import AsyncSupersetClient\n\n client = AsyncSupersetClient(env)\n try:\n dashboard_id = await _find_dashboard_id_by_slug_async(client, normalized_ref)\n if dashboard_id is None:\n raise HTTPException(\n status_code=404, detail=f\"Dashboard slug '{normalized_ref}' not found\"\n )\n return dashboard_id\n finally:\n await client.aclose()\n# [/DEF:_resolve_dashboard_id_from_ref_async:Function]\n\n\n# [DEF:_resolve_repo_key_from_ref:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback.\ndef _resolve_repo_key_from_ref(\n dashboard_ref: str,\n dashboard_id: int,\n config_manager,\n env_id: Optional[str] = None,\n) -> str:\n normalized_ref = str(dashboard_ref or \"\").strip()\n if normalized_ref and not normalized_ref.isdigit():\n return normalized_ref\n\n if env_id:\n try:\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if env:\n payload = SupersetClient(env).get_dashboard(dashboard_id)\n dashboard_data = (\n payload.get(\"result\", payload) if isinstance(payload, dict) else {}\n )\n dashboard_slug = dashboard_data.get(\"slug\")\n if dashboard_slug:\n return str(dashboard_slug)\n except Exception:\n pass\n\n return f\"dashboard-{dashboard_id}\"\n# [/DEF:_resolve_repo_key_from_ref:Function]\n\n\n# [DEF:_sanitize_optional_identity_value:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Normalize optional identity value into trimmed string or None.\ndef _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]:\n normalized = str(value or \"\").strip()\n if not normalized:\n return None\n return normalized\n# [/DEF:_sanitize_optional_identity_value:Function]\n\n\n# [DEF:_resolve_current_user_git_identity:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve configured Git username/email from current user's profile preferences.\ndef _resolve_current_user_git_identity(\n db: Session,\n current_user: Optional[User],\n) -> Optional[tuple[str, str]]:\n if db is None or not hasattr(db, \"query\"):\n return None\n\n user_id = _sanitize_optional_identity_value(getattr(current_user, \"id\", None))\n if not user_id:\n return None\n\n try:\n preference = (\n db.query(UserDashboardPreference)\n .filter(UserDashboardPreference.user_id == user_id)\n .first()\n )\n except Exception as resolve_error:\n logger.warning(\n \"[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s\",\n user_id,\n resolve_error,\n )\n return None\n\n if not preference:\n return None\n\n git_username = _sanitize_optional_identity_value(\n getattr(preference, \"git_username\", None)\n )\n git_email = _sanitize_optional_identity_value(\n getattr(preference, \"git_email\", None)\n )\n if not git_username or not git_email:\n return None\n return git_username, git_email\n# [/DEF:_resolve_current_user_git_identity:Function]\n\n\n# [DEF:_apply_git_identity_from_profile:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Apply user-scoped Git identity to repository-local config before write/pull operations.\ndef _apply_git_identity_from_profile(\n dashboard_id: int,\n db: Session,\n current_user: Optional[User],\n) -> None:\n identity = _resolve_current_user_git_identity(db, current_user)\n if not identity:\n return\n\n git_service = get_git_service()\n configure_identity = getattr(git_service, \"configure_identity\", None)\n if not callable(configure_identity):\n return\n\n git_username, git_email = identity\n configure_identity(dashboard_id, git_username, git_email)\n# [/DEF:_apply_git_identity_from_profile:Function]\n# [/DEF:GitHelpers:Module]\n" + "body": "# [DEF:GitHelpers:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: Shared helper functions for Git route modules.\n# @LAYER: API\n# @SEMANTICS: git, helpers, resolution, identity\n# @RELATION USES -> [GitDeps]\n# @RELATION USES -> [SupersetClient]\n# @RELATION USES -> [UserDashboardPreference]\n\nimport os\nfrom typing import Optional\n\nfrom fastapi import HTTPException\nfrom sqlalchemy.orm import Session\n\nfrom src.core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"GitHelpers\")\nfrom src.core.superset_client import SupersetClient\nfrom src.dependencies import get_config_manager\nfrom src.models.auth import User\nfrom src.models.git import GitServerConfig\nfrom src.models.profile import UserDashboardPreference\n\nfrom ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH\n\n\n# [DEF:_build_no_repo_status_payload:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Build a consistent status payload for dashboards without initialized repositories.\n# @POST: Returns a stable payload compatible with frontend repository status parsing.\ndef _build_no_repo_status_payload() -> dict:\n return {\n \"is_dirty\": False,\n \"untracked_files\": [],\n \"modified_files\": [],\n \"staged_files\": [],\n \"current_branch\": None,\n \"upstream_branch\": None,\n \"has_upstream\": False,\n \"ahead_count\": 0,\n \"behind_count\": 0,\n \"is_diverged\": False,\n \"sync_state\": \"NO_REPO\",\n \"sync_status\": \"NO_REPO\",\n \"has_repo\": False,\n }\n# [/DEF:_build_no_repo_status_payload:Function]\n\n\n# [DEF:_handle_unexpected_git_route_error:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses.\n# @PRE: `error` is a non-HTTPException instance.\n# @POST: Raises HTTPException(500) with route-specific context.\ndef _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None:\n log.explore(f\"{route_name} failed: {error}\", error=str(error))\n raise HTTPException(status_code=500, detail=f\"{route_name} failed: {str(error)}\")\n# [/DEF:_handle_unexpected_git_route_error:Function]\n\n\n# [DEF:_resolve_repository_status:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics.\n# @PRE: `dashboard_id` is a valid integer.\n# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent.\ndef _resolve_repository_status(dashboard_id: int) -> dict:\n git_service = get_git_service()\n repo_path = git_service._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n log.reason(\n f\"Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n\n try:\n return git_service.get_status(dashboard_id)\n except HTTPException as e:\n if e.status_code == 404:\n log.reason(\n f\"Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n raise\n# [/DEF:_resolve_repository_status:Function]\n\n\n# [DEF:_get_git_config_or_404:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve GitServerConfig by id or raise 404.\ndef _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig:\n config = db.query(GitServerConfig).filter(GitServerConfig.id == config_id).first()\n if not config:\n raise HTTPException(status_code=404, detail=\"Git configuration not found\")\n return config\n# [/DEF:_get_git_config_or_404:Function]\n\n\n# [DEF:_find_dashboard_id_by_slug:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard numeric ID by slug in a specific environment.\ndef _find_dashboard_id_by_slug(\n client: SupersetClient,\n dashboard_slug: str,\n) -> Optional[int]:\n query_variants = [\n {\n \"filters\": [{\"col\": \"slug\", \"opr\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n {\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n ]\n\n for query in query_variants:\n try:\n _count, dashboards = client.get_dashboards_page(query=query)\n if dashboards:\n resolved_id = dashboards[0].get(\"id\")\n if resolved_id is not None:\n return int(resolved_id)\n except Exception:\n continue\n return None\n# [/DEF:_find_dashboard_id_by_slug:Function]\n\n\n# [DEF:_resolve_dashboard_id_from_ref:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard ID from slug-or-id reference for Git routes.\ndef _resolve_dashboard_id_from_ref(\n dashboard_ref: str,\n config_manager,\n env_id: Optional[str] = None,\n) -> int:\n normalized_ref = str(dashboard_ref or \"\").strip()\n if not normalized_ref:\n raise HTTPException(status_code=400, detail=\"dashboard_ref is required\")\n\n if normalized_ref.isdigit():\n return int(normalized_ref)\n\n if not env_id:\n raise HTTPException(\n status_code=400,\n detail=\"env_id is required for slug-based Git operations\",\n )\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n dashboard_id = _find_dashboard_id_by_slug(SupersetClient(env), normalized_ref)\n if dashboard_id is None:\n raise HTTPException(\n status_code=404, detail=f\"Dashboard slug '{normalized_ref}' not found\"\n )\n return dashboard_id\n# [/DEF:_resolve_dashboard_id_from_ref:Function]\n\n\n# [DEF:_find_dashboard_id_by_slug_async:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes.\nasync def _find_dashboard_id_by_slug_async(\n client: \"AsyncSupersetClient\",\n dashboard_slug: str,\n) -> Optional[int]:\n query_variants = [\n {\n \"filters\": [{\"col\": \"slug\", \"opr\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n {\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dashboard_slug}],\n \"page\": 0,\n \"page_size\": 1,\n },\n ]\n\n for query in query_variants:\n try:\n _count, dashboards = await client.get_dashboards_page_async(query=query)\n if dashboards:\n resolved_id = dashboards[0].get(\"id\")\n if resolved_id is not None:\n return int(resolved_id)\n except Exception:\n continue\n return None\n# [/DEF:_find_dashboard_id_by_slug_async:Function]\n\n\n# [DEF:_resolve_dashboard_id_from_ref_async:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes.\nasync def _resolve_dashboard_id_from_ref_async(\n dashboard_ref: str,\n config_manager,\n env_id: Optional[str] = None,\n) -> int:\n normalized_ref = str(dashboard_ref or \"\").strip()\n if not normalized_ref:\n raise HTTPException(status_code=400, detail=\"dashboard_ref is required\")\n\n if normalized_ref.isdigit():\n return int(normalized_ref)\n\n if not env_id:\n raise HTTPException(\n status_code=400,\n detail=\"env_id is required for slug-based Git operations\",\n )\n\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if not env:\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n from src.core.async_superset_client import AsyncSupersetClient\n\n client = AsyncSupersetClient(env)\n try:\n dashboard_id = await _find_dashboard_id_by_slug_async(client, normalized_ref)\n if dashboard_id is None:\n raise HTTPException(\n status_code=404, detail=f\"Dashboard slug '{normalized_ref}' not found\"\n )\n return dashboard_id\n finally:\n await client.aclose()\n# [/DEF:_resolve_dashboard_id_from_ref_async:Function]\n\n\n# [DEF:_resolve_repo_key_from_ref:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback.\ndef _resolve_repo_key_from_ref(\n dashboard_ref: str,\n dashboard_id: int,\n config_manager,\n env_id: Optional[str] = None,\n) -> str:\n normalized_ref = str(dashboard_ref or \"\").strip()\n if normalized_ref and not normalized_ref.isdigit():\n return normalized_ref\n\n if env_id:\n try:\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n if env:\n payload = SupersetClient(env).get_dashboard(dashboard_id)\n dashboard_data = (\n payload.get(\"result\", payload) if isinstance(payload, dict) else {}\n )\n dashboard_slug = dashboard_data.get(\"slug\")\n if dashboard_slug:\n return str(dashboard_slug)\n except Exception:\n pass\n\n return f\"dashboard-{dashboard_id}\"\n# [/DEF:_resolve_repo_key_from_ref:Function]\n\n\n# [DEF:_sanitize_optional_identity_value:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Normalize optional identity value into trimmed string or None.\ndef _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]:\n normalized = str(value or \"\").strip()\n if not normalized:\n return None\n return normalized\n# [/DEF:_sanitize_optional_identity_value:Function]\n\n\n# [DEF:_resolve_current_user_git_identity:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve configured Git username/email from current user's profile preferences.\ndef _resolve_current_user_git_identity(\n db: Session,\n current_user: Optional[User],\n) -> Optional[tuple[str, str]]:\n if db is None or not hasattr(db, \"query\"):\n return None\n\n user_id = _sanitize_optional_identity_value(getattr(current_user, \"id\", None))\n if not user_id:\n return None\n\n try:\n preference = (\n db.query(UserDashboardPreference)\n .filter(UserDashboardPreference.user_id == user_id)\n .first()\n )\n except Exception as resolve_error:\n log.explore(f\"Failed to load profile preference for user {user_id}: {resolve_error}\", error=str(resolve_error))\n return None\n\n if not preference:\n return None\n\n git_username = _sanitize_optional_identity_value(\n getattr(preference, \"git_username\", None)\n )\n git_email = _sanitize_optional_identity_value(\n getattr(preference, \"git_email\", None)\n )\n if not git_username or not git_email:\n return None\n return git_username, git_email\n# [/DEF:_resolve_current_user_git_identity:Function]\n\n\n# [DEF:_apply_git_identity_from_profile:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Apply user-scoped Git identity to repository-local config before write/pull operations.\ndef _apply_git_identity_from_profile(\n dashboard_id: int,\n db: Session,\n current_user: Optional[User],\n) -> None:\n identity = _resolve_current_user_git_identity(db, current_user)\n if not identity:\n return\n\n git_service = get_git_service()\n configure_identity = getattr(git_service, \"configure_identity\", None)\n if not callable(configure_identity):\n return\n\n git_username, git_email = identity\n configure_identity(dashboard_id, git_username, git_email)\n# [/DEF:_apply_git_identity_from_profile:Function]\n# [/DEF:GitHelpers:Module]\n" }, { "contract_id": "_build_no_repo_status_payload", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 26, - "end_line": 46, + "start_line": 28, + "end_line": 48, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -18958,8 +18958,8 @@ "contract_id": "_handle_unexpected_git_route_error", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 49, - "end_line": 57, + "start_line": 51, + "end_line": 59, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -18990,14 +18990,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:_handle_unexpected_git_route_error:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses.\n# @PRE: `error` is a non-HTTPException instance.\n# @POST: Raises HTTPException(500) with route-specific context.\ndef _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None:\n logger.error(f\"[{route_name}][Coherence:Failed] {error}\")\n raise HTTPException(status_code=500, detail=f\"{route_name} failed: {str(error)}\")\n# [/DEF:_handle_unexpected_git_route_error:Function]\n" + "body": "# [DEF:_handle_unexpected_git_route_error:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses.\n# @PRE: `error` is a non-HTTPException instance.\n# @POST: Raises HTTPException(500) with route-specific context.\ndef _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None:\n log.explore(f\"{route_name} failed: {error}\", error=str(error))\n raise HTTPException(status_code=500, detail=f\"{route_name} failed: {str(error)}\")\n# [/DEF:_handle_unexpected_git_route_error:Function]\n" }, { "contract_id": "_resolve_repository_status", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 60, - "end_line": 83, + "start_line": 62, + "end_line": 85, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19028,14 +19028,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:_resolve_repository_status:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics.\n# @PRE: `dashboard_id` is a valid integer.\n# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent.\ndef _resolve_repository_status(dashboard_id: int) -> dict:\n git_service = get_git_service()\n repo_path = git_service._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n logger.debug(\n f\"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n\n try:\n return git_service.get_status(dashboard_id)\n except HTTPException as e:\n if e.status_code == 404:\n logger.debug(\n f\"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n raise\n# [/DEF:_resolve_repository_status:Function]\n" + "body": "# [DEF:_resolve_repository_status:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics.\n# @PRE: `dashboard_id` is a valid integer.\n# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent.\ndef _resolve_repository_status(dashboard_id: int) -> dict:\n git_service = get_git_service()\n repo_path = git_service._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n log.reason(\n f\"Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n\n try:\n return git_service.get_status(dashboard_id)\n except HTTPException as e:\n if e.status_code == 404:\n log.reason(\n f\"Repository is not initialized for dashboard {dashboard_id}\"\n )\n return _build_no_repo_status_payload()\n raise\n# [/DEF:_resolve_repository_status:Function]\n" }, { "contract_id": "_get_git_config_or_404", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 86, - "end_line": 94, + "start_line": 88, + "end_line": 96, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19051,8 +19051,8 @@ "contract_id": "_resolve_repo_key_from_ref", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 239, - "end_line": 268, + "start_line": 241, + "end_line": 270, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19068,8 +19068,8 @@ "contract_id": "_sanitize_optional_identity_value", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 271, - "end_line": 279, + "start_line": 273, + "end_line": 281, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -19085,8 +19085,8 @@ "contract_id": "_resolve_current_user_git_identity", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 282, - "end_line": 322, + "start_line": 284, + "end_line": 320, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19096,14 +19096,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:_resolve_current_user_git_identity:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve configured Git username/email from current user's profile preferences.\ndef _resolve_current_user_git_identity(\n db: Session,\n current_user: Optional[User],\n) -> Optional[tuple[str, str]]:\n if db is None or not hasattr(db, \"query\"):\n return None\n\n user_id = _sanitize_optional_identity_value(getattr(current_user, \"id\", None))\n if not user_id:\n return None\n\n try:\n preference = (\n db.query(UserDashboardPreference)\n .filter(UserDashboardPreference.user_id == user_id)\n .first()\n )\n except Exception as resolve_error:\n logger.warning(\n \"[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s\",\n user_id,\n resolve_error,\n )\n return None\n\n if not preference:\n return None\n\n git_username = _sanitize_optional_identity_value(\n getattr(preference, \"git_username\", None)\n )\n git_email = _sanitize_optional_identity_value(\n getattr(preference, \"git_email\", None)\n )\n if not git_username or not git_email:\n return None\n return git_username, git_email\n# [/DEF:_resolve_current_user_git_identity:Function]\n" + "body": "# [DEF:_resolve_current_user_git_identity:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve configured Git username/email from current user's profile preferences.\ndef _resolve_current_user_git_identity(\n db: Session,\n current_user: Optional[User],\n) -> Optional[tuple[str, str]]:\n if db is None or not hasattr(db, \"query\"):\n return None\n\n user_id = _sanitize_optional_identity_value(getattr(current_user, \"id\", None))\n if not user_id:\n return None\n\n try:\n preference = (\n db.query(UserDashboardPreference)\n .filter(UserDashboardPreference.user_id == user_id)\n .first()\n )\n except Exception as resolve_error:\n log.explore(f\"Failed to load profile preference for user {user_id}: {resolve_error}\", error=str(resolve_error))\n return None\n\n if not preference:\n return None\n\n git_username = _sanitize_optional_identity_value(\n getattr(preference, \"git_username\", None)\n )\n git_email = _sanitize_optional_identity_value(\n getattr(preference, \"git_email\", None)\n )\n if not git_username or not git_email:\n return None\n return git_username, git_email\n# [/DEF:_resolve_current_user_git_identity:Function]\n" }, { "contract_id": "_apply_git_identity_from_profile", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_helpers.py", - "start_line": 325, - "end_line": 344, + "start_line": 323, + "end_line": 342, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19388,7 +19388,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", "start_line": 1, - "end_line": 365, + "end_line": 352, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -19432,14 +19432,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitRepoOperationsRoutes:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history).\n# @LAYER: API\n# @SEMANTICS: git, repository, operations, commit, push, pull, status\n\nfrom typing import List, Optional\n\nfrom fastapi import Depends, HTTPException\nfrom sqlalchemy.orm import Session\n\nfrom src.core.database import get_db\nfrom src.core.logger import belief_scope, logger\nfrom src.dependencies import get_config_manager, get_current_user, has_permission\nfrom src.models.auth import User\nfrom src.models.git import GitRepository, GitServerConfig\n\nfrom src.api.routes.git_schemas import (\n CommitCreate,\n CommitSchema,\n RepoStatusBatchRequest,\n RepoStatusBatchResponse,\n)\n\nfrom ._router import router\nfrom ._helpers import (\n _apply_git_identity_from_profile,\n _build_no_repo_status_payload,\n _handle_unexpected_git_route_error,\n _resolve_repository_status,\n)\nfrom ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH\n\n\n# [DEF:commit_changes:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Stage and commit changes in the dashboard's repository.\n@router.post(\"/repositories/{dashboard_ref}/commit\")\nasync def commit_changes(\n dashboard_ref: str,\n commit_data: CommitCreate,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"commit_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.commit_changes(\n dashboard_id, commit_data.message, commit_data.files\n )\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"commit_changes\", e)\n# [/DEF:commit_changes:Function]\n\n\n# [DEF:push_changes:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Push local commits to the remote repository.\n@router.post(\"/repositories/{dashboard_ref}/push\")\nasync def push_changes(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"push_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _gs.push_changes(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"push_changes\", e)\n# [/DEF:push_changes:Function]\n\n\n# [DEF:pull_changes:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Pull changes from the remote repository.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/pull\")\nasync def pull_changes(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"pull_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n db_repo = None\n config_url = None\n config_provider = None\n try:\n db_repo_candidate = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if getattr(db_repo_candidate, \"config_id\", None):\n db_repo = db_repo_candidate\n config_row = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == db_repo.config_id)\n .first()\n )\n if config_row:\n config_url = config_row.url\n config_provider = config_row.provider\n except Exception as diagnostics_error:\n logger.warning(\n \"[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s\",\n dashboard_id,\n diagnostics_error,\n )\n logger.info(\n \"[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s \"\n \"binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s\",\n dashboard_ref,\n env_id,\n dashboard_id,\n bool(db_repo),\n (db_repo.local_path if db_repo else None),\n (db_repo.remote_url if db_repo else None),\n (db_repo.config_id if db_repo else None),\n config_provider,\n config_url,\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.pull_changes(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"pull_changes\", e)\n# [/DEF:pull_changes:Function]\n\n\n# [DEF:get_repository_status:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get current Git status for a dashboard repository.\n@router.get(\"/repositories/{dashboard_ref}/status\")\nasync def get_repository_status(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_status\"):\n from . import _resolve_dashboard_id_from_ref_async\n\n try:\n dashboard_id = await _resolve_dashboard_id_from_ref_async(\n dashboard_ref, config_manager, env_id\n )\n return _resolve_repository_status(dashboard_id)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_repository_status\", e)\n# [/DEF:get_repository_status:Function]\n\n\n# [DEF:get_repository_status_batch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request.\n@router.post(\"/repositories/status/batch\", response_model=RepoStatusBatchResponse)\nasync def get_repository_status_batch(\n request: RepoStatusBatchRequest,\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_status_batch\"):\n dashboard_ids = list(dict.fromkeys(request.dashboard_ids))\n if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH:\n logger.warning(\n \"[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.\",\n len(dashboard_ids),\n MAX_REPOSITORY_STATUS_BATCH,\n )\n dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH]\n\n statuses = {}\n for dashboard_id in dashboard_ids:\n try:\n statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id)\n except HTTPException:\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n except Exception as e:\n logger.error(\n f\"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}\"\n )\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n return RepoStatusBatchResponse(statuses=statuses)\n# [/DEF:get_repository_status_batch:Function]\n\n\n# [DEF:get_repository_diff:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get Git diff for a dashboard repository.\n@router.get(\"/repositories/{dashboard_ref}/diff\")\nasync def get_repository_diff(\n dashboard_ref: str,\n file_path: Optional[str] = None,\n staged: bool = False,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"get_repository_diff\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n return _gs.get_diff(dashboard_id, file_path, staged)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_repository_diff\", e)\n# [/DEF:get_repository_diff:Function]\n\n\n# [DEF:get_history:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: View commit history for a dashboard's repository.\n@router.get(\"/repositories/{dashboard_ref}/history\", response_model=List[CommitSchema])\nasync def get_history(\n dashboard_ref: str,\n limit: int = 50,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"get_history\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n return _gs.get_commit_history(dashboard_id, limit)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_history\", e)\n# [/DEF:get_history:Function]\n\n\n# [DEF:generate_commit_message:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Generate a suggested commit message using LLM.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/generate-message\")\nasync def generate_commit_message(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"generate_commit_message\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n diff = _gs.get_diff(dashboard_id, staged=True)\n if not diff:\n diff = _gs.get_diff(dashboard_id, staged=False)\n\n if not diff:\n return {\"message\": \"No changes detected\"}\n\n history_objs = _gs.get_commit_history(dashboard_id, limit=5)\n history = [h.message for h in history_objs if hasattr(h, \"message\")]\n\n from ...services.llm_provider import LLMProviderService\n from ...plugins.llm_analysis.service import LLMClient\n from ...plugins.llm_analysis.models import LLMProviderType\n from ...services.llm_prompt_templates import (\n DEFAULT_LLM_PROMPTS,\n normalize_llm_settings,\n resolve_bound_provider_id,\n )\n\n llm_service = LLMProviderService(db)\n providers = llm_service.get_all_providers()\n llm_settings = normalize_llm_settings(\n config_manager.get_config().settings.llm\n )\n bound_provider_id = resolve_bound_provider_id(llm_settings, \"git_commit\")\n provider = next((p for p in providers if p.id == bound_provider_id), None)\n if not provider:\n provider = next((p for p in providers if p.is_active), None)\n\n if not provider:\n raise HTTPException(\n status_code=400, detail=\"No active LLM provider found\"\n )\n\n api_key = llm_service.get_decrypted_api_key(provider.id)\n client = LLMClient(\n provider_type=LLMProviderType(provider.provider_type),\n api_key=api_key,\n base_url=provider.base_url,\n default_model=provider.default_model,\n )\n\n from ...plugins.git.llm_extension import GitLLMExtension\n\n extension = GitLLMExtension(client)\n git_prompt = llm_settings[\"prompts\"].get(\n \"git_commit_prompt\",\n DEFAULT_LLM_PROMPTS[\"git_commit_prompt\"],\n )\n message = await extension.suggest_commit_message(\n diff,\n history,\n prompt_template=git_prompt,\n )\n return {\"message\": message}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"generate_commit_message\", e)\n# [/DEF:generate_commit_message:Function]\n# [/DEF:GitRepoOperationsRoutes:Module]\n" + "body": "# [DEF:GitRepoOperationsRoutes:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history).\n# @LAYER: API\n# @SEMANTICS: git, repository, operations, commit, push, pull, status\n\nfrom typing import List, Optional\n\nfrom fastapi import Depends, HTTPException\nfrom sqlalchemy.orm import Session\n\nfrom src.core.database import get_db\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\n\nlog = MarkerLogger(\"GitRepoOps\")\nfrom src.dependencies import get_config_manager, get_current_user, has_permission\nfrom src.models.auth import User\nfrom src.models.git import GitRepository, GitServerConfig\n\nfrom src.api.routes.git_schemas import (\n CommitCreate,\n CommitSchema,\n RepoStatusBatchRequest,\n RepoStatusBatchResponse,\n)\n\nfrom ._router import router\nfrom ._helpers import (\n _apply_git_identity_from_profile,\n _build_no_repo_status_payload,\n _handle_unexpected_git_route_error,\n _resolve_repository_status,\n)\nfrom ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH\n\n\n# [DEF:commit_changes:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Stage and commit changes in the dashboard's repository.\n@router.post(\"/repositories/{dashboard_ref}/commit\")\nasync def commit_changes(\n dashboard_ref: str,\n commit_data: CommitCreate,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"commit_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.commit_changes(\n dashboard_id, commit_data.message, commit_data.files\n )\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"commit_changes\", e)\n# [/DEF:commit_changes:Function]\n\n\n# [DEF:push_changes:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Push local commits to the remote repository.\n@router.post(\"/repositories/{dashboard_ref}/push\")\nasync def push_changes(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"push_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _gs.push_changes(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"push_changes\", e)\n# [/DEF:push_changes:Function]\n\n\n# [DEF:pull_changes:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Pull changes from the remote repository.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/pull\")\nasync def pull_changes(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"pull_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n db_repo = None\n config_url = None\n config_provider = None\n try:\n db_repo_candidate = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if getattr(db_repo_candidate, \"config_id\", None):\n db_repo = db_repo_candidate\n config_row = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == db_repo.config_id)\n .first()\n )\n if config_row:\n config_url = config_row.url\n config_provider = config_row.provider\n except Exception as diagnostics_error:\n log.explore(f\"Failed to load repository binding diagnostics for dashboard {dashboard_id}: {diagnostics_error}\", error=str(diagnostics_error))\n log.reason(\n f\"Route diagnostics dashboard_ref={dashboard_ref} env_id={env_id} resolved_dashboard_id={dashboard_id} \"\n f\"binding_exists={bool(db_repo)} binding_local_path={(db_repo.local_path if db_repo else None)} \"\n f\"binding_remote_url={(db_repo.remote_url if db_repo else None)} \"\n f\"binding_config_id={(db_repo.config_id if db_repo else None)} \"\n f\"config_provider={config_provider} config_url={config_url}\"\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.pull_changes(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"pull_changes\", e)\n# [/DEF:pull_changes:Function]\n\n\n# [DEF:get_repository_status:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get current Git status for a dashboard repository.\n@router.get(\"/repositories/{dashboard_ref}/status\")\nasync def get_repository_status(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_status\"):\n from . import _resolve_dashboard_id_from_ref_async\n\n try:\n dashboard_id = await _resolve_dashboard_id_from_ref_async(\n dashboard_ref, config_manager, env_id\n )\n return _resolve_repository_status(dashboard_id)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_repository_status\", e)\n# [/DEF:get_repository_status:Function]\n\n\n# [DEF:get_repository_status_batch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request.\n@router.post(\"/repositories/status/batch\", response_model=RepoStatusBatchResponse)\nasync def get_repository_status_batch(\n request: RepoStatusBatchRequest,\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_status_batch\"):\n dashboard_ids = list(dict.fromkeys(request.dashboard_ids))\n if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH:\n log.explore(f\"Batch size {len(dashboard_ids)} exceeds limit {MAX_REPOSITORY_STATUS_BATCH}. Truncating request.\", error=\"Batch size exceeds limit\")\n dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH]\n\n statuses = {}\n for dashboard_id in dashboard_ids:\n try:\n statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id)\n except HTTPException:\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n except Exception as e:\n log.explore(f\"Failed for dashboard {dashboard_id}: {e}\", error=str(e))\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n return RepoStatusBatchResponse(statuses=statuses)\n# [/DEF:get_repository_status_batch:Function]\n\n\n# [DEF:get_repository_diff:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get Git diff for a dashboard repository.\n@router.get(\"/repositories/{dashboard_ref}/diff\")\nasync def get_repository_diff(\n dashboard_ref: str,\n file_path: Optional[str] = None,\n staged: bool = False,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"get_repository_diff\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n return _gs.get_diff(dashboard_id, file_path, staged)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_repository_diff\", e)\n# [/DEF:get_repository_diff:Function]\n\n\n# [DEF:get_history:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: View commit history for a dashboard's repository.\n@router.get(\"/repositories/{dashboard_ref}/history\", response_model=List[CommitSchema])\nasync def get_history(\n dashboard_ref: str,\n limit: int = 50,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"get_history\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n return _gs.get_commit_history(dashboard_id, limit)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_history\", e)\n# [/DEF:get_history:Function]\n\n\n# [DEF:generate_commit_message:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Generate a suggested commit message using LLM.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/generate-message\")\nasync def generate_commit_message(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"generate_commit_message\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n diff = _gs.get_diff(dashboard_id, staged=True)\n if not diff:\n diff = _gs.get_diff(dashboard_id, staged=False)\n\n if not diff:\n return {\"message\": \"No changes detected\"}\n\n history_objs = _gs.get_commit_history(dashboard_id, limit=5)\n history = [h.message for h in history_objs if hasattr(h, \"message\")]\n\n from ...services.llm_provider import LLMProviderService\n from ...plugins.llm_analysis.service import LLMClient\n from ...plugins.llm_analysis.models import LLMProviderType\n from ...services.llm_prompt_templates import (\n DEFAULT_LLM_PROMPTS,\n normalize_llm_settings,\n resolve_bound_provider_id,\n )\n\n llm_service = LLMProviderService(db)\n providers = llm_service.get_all_providers()\n llm_settings = normalize_llm_settings(\n config_manager.get_config().settings.llm\n )\n bound_provider_id = resolve_bound_provider_id(llm_settings, \"git_commit\")\n provider = next((p for p in providers if p.id == bound_provider_id), None)\n if not provider:\n provider = next((p for p in providers if p.is_active), None)\n\n if not provider:\n raise HTTPException(\n status_code=400, detail=\"No active LLM provider found\"\n )\n\n api_key = llm_service.get_decrypted_api_key(provider.id)\n client = LLMClient(\n provider_type=LLMProviderType(provider.provider_type),\n api_key=api_key,\n base_url=provider.base_url,\n default_model=provider.default_model,\n )\n\n from ...plugins.git.llm_extension import GitLLMExtension\n\n extension = GitLLMExtension(client)\n git_prompt = llm_settings[\"prompts\"].get(\n \"git_commit_prompt\",\n DEFAULT_LLM_PROMPTS[\"git_commit_prompt\"],\n )\n message = await extension.suggest_commit_message(\n diff,\n history,\n prompt_template=git_prompt,\n )\n return {\"message\": message}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"generate_commit_message\", e)\n# [/DEF:generate_commit_message:Function]\n# [/DEF:GitRepoOperationsRoutes:Module]\n" }, { "contract_id": "commit_changes", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 35, - "end_line": 65, + "start_line": 38, + "end_line": 68, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19455,8 +19455,8 @@ "contract_id": "push_changes", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 68, - "end_line": 92, + "start_line": 71, + "end_line": 95, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19472,8 +19472,8 @@ "contract_id": "pull_changes", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 95, - "end_line": 161, + "start_line": 98, + "end_line": 154, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -19490,14 +19490,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:pull_changes:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Pull changes from the remote repository.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/pull\")\nasync def pull_changes(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"pull_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n db_repo = None\n config_url = None\n config_provider = None\n try:\n db_repo_candidate = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if getattr(db_repo_candidate, \"config_id\", None):\n db_repo = db_repo_candidate\n config_row = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == db_repo.config_id)\n .first()\n )\n if config_row:\n config_url = config_row.url\n config_provider = config_row.provider\n except Exception as diagnostics_error:\n logger.warning(\n \"[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s\",\n dashboard_id,\n diagnostics_error,\n )\n logger.info(\n \"[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s \"\n \"binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s\",\n dashboard_ref,\n env_id,\n dashboard_id,\n bool(db_repo),\n (db_repo.local_path if db_repo else None),\n (db_repo.remote_url if db_repo else None),\n (db_repo.config_id if db_repo else None),\n config_provider,\n config_url,\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.pull_changes(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"pull_changes\", e)\n# [/DEF:pull_changes:Function]\n" + "body": "# [DEF:pull_changes:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Pull changes from the remote repository.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/pull\")\nasync def pull_changes(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"pull_changes\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n db_repo = None\n config_url = None\n config_provider = None\n try:\n db_repo_candidate = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if getattr(db_repo_candidate, \"config_id\", None):\n db_repo = db_repo_candidate\n config_row = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == db_repo.config_id)\n .first()\n )\n if config_row:\n config_url = config_row.url\n config_provider = config_row.provider\n except Exception as diagnostics_error:\n log.explore(f\"Failed to load repository binding diagnostics for dashboard {dashboard_id}: {diagnostics_error}\", error=str(diagnostics_error))\n log.reason(\n f\"Route diagnostics dashboard_ref={dashboard_ref} env_id={env_id} resolved_dashboard_id={dashboard_id} \"\n f\"binding_exists={bool(db_repo)} binding_local_path={(db_repo.local_path if db_repo else None)} \"\n f\"binding_remote_url={(db_repo.remote_url if db_repo else None)} \"\n f\"binding_config_id={(db_repo.config_id if db_repo else None)} \"\n f\"config_provider={config_provider} config_url={config_url}\"\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.pull_changes(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"pull_changes\", e)\n# [/DEF:pull_changes:Function]\n" }, { "contract_id": "get_repository_status", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 164, - "end_line": 186, + "start_line": 157, + "end_line": 179, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19513,8 +19513,8 @@ "contract_id": "get_repository_status_batch", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 189, - "end_line": 227, + "start_line": 182, + "end_line": 214, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19524,14 +19524,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:get_repository_status_batch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request.\n@router.post(\"/repositories/status/batch\", response_model=RepoStatusBatchResponse)\nasync def get_repository_status_batch(\n request: RepoStatusBatchRequest,\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_status_batch\"):\n dashboard_ids = list(dict.fromkeys(request.dashboard_ids))\n if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH:\n logger.warning(\n \"[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.\",\n len(dashboard_ids),\n MAX_REPOSITORY_STATUS_BATCH,\n )\n dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH]\n\n statuses = {}\n for dashboard_id in dashboard_ids:\n try:\n statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id)\n except HTTPException:\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n except Exception as e:\n logger.error(\n f\"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}\"\n )\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n return RepoStatusBatchResponse(statuses=statuses)\n# [/DEF:get_repository_status_batch:Function]\n" + "body": "# [DEF:get_repository_status_batch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Get Git statuses for multiple dashboard repositories in one request.\n@router.post(\"/repositories/status/batch\", response_model=RepoStatusBatchResponse)\nasync def get_repository_status_batch(\n request: RepoStatusBatchRequest,\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_status_batch\"):\n dashboard_ids = list(dict.fromkeys(request.dashboard_ids))\n if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH:\n log.explore(f\"Batch size {len(dashboard_ids)} exceeds limit {MAX_REPOSITORY_STATUS_BATCH}. Truncating request.\", error=\"Batch size exceeds limit\")\n dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH]\n\n statuses = {}\n for dashboard_id in dashboard_ids:\n try:\n statuses[str(dashboard_id)] = _resolve_repository_status(dashboard_id)\n except HTTPException:\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n except Exception as e:\n log.explore(f\"Failed for dashboard {dashboard_id}: {e}\", error=str(e))\n statuses[str(dashboard_id)] = {\n **_build_no_repo_status_payload(),\n \"sync_state\": \"ERROR\",\n \"sync_status\": \"ERROR\",\n }\n return RepoStatusBatchResponse(statuses=statuses)\n# [/DEF:get_repository_status_batch:Function]\n" }, { "contract_id": "get_repository_diff", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 230, - "end_line": 255, + "start_line": 217, + "end_line": 242, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19547,8 +19547,8 @@ "contract_id": "generate_commit_message", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_operations_routes.py", - "start_line": 285, - "end_line": 364, + "start_line": 272, + "end_line": 351, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -19572,7 +19572,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/git/_repo_routes.py", "start_line": 1, - "end_line": 269, + "end_line": 265, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -19614,14 +19614,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitRepoRoutes:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: FastAPI endpoints for core Git repository operations (init, binding, branches, checkout).\n# @LAYER: API\n# @SEMANTICS: git, repository, routes, init, branches\n\nfrom typing import List, Optional\n\nfrom fastapi import Depends, HTTPException\nfrom sqlalchemy.orm import Session\n\nfrom src.core.database import get_db\nfrom src.core.logger import belief_scope, logger\nfrom src.dependencies import get_config_manager, get_current_user, has_permission\nfrom src.models.auth import User\nfrom src.models.git import GitRepository, GitServerConfig\n\nfrom src.api.routes.git_schemas import (\n BranchCheckout,\n BranchCreate,\n BranchSchema,\n RepoInitRequest,\n RepositoryBindingSchema,\n)\n\nfrom ._router import router\nfrom ._helpers import (\n _apply_git_identity_from_profile,\n _get_git_config_or_404,\n _handle_unexpected_git_route_error,\n)\nfrom ._deps import get_git_service\n\n\n# [DEF:init_repository:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/init\")\nasync def init_repository(\n dashboard_ref: str,\n init_data: RepoInitRequest,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"init_repository\"):\n from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref\n\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n repo_key = _resolve_repo_key_from_ref(\n dashboard_ref, dashboard_id, config_manager, env_id\n )\n config = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == init_data.config_id)\n .first()\n )\n if not config:\n raise HTTPException(status_code=404, detail=\"Git configuration not found\")\n\n try:\n logger.info(\n f\"[init_repository][Action] Initializing repo for dashboard {dashboard_id}\"\n )\n _gs.init_repo(\n dashboard_id,\n init_data.remote_url,\n config.pat,\n repo_key=repo_key,\n default_branch=config.default_branch,\n )\n\n repo_path = _gs._get_repo_path(dashboard_id, repo_key=repo_key)\n db_repo = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if not db_repo:\n db_repo = GitRepository(\n dashboard_id=dashboard_id,\n config_id=config.id,\n remote_url=init_data.remote_url,\n local_path=repo_path,\n current_branch=\"dev\",\n )\n db.add(db_repo)\n else:\n db_repo.config_id = config.id\n db_repo.remote_url = init_data.remote_url\n db_repo.local_path = repo_path\n db_repo.current_branch = \"dev\"\n\n db.commit()\n logger.info(\n f\"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}\"\n )\n return {\"status\": \"success\", \"message\": \"Repository initialized\"}\n except Exception as e:\n db.rollback()\n logger.error(\n f\"[init_repository][Coherence:Failed] Failed to init repository: {e}\"\n )\n if isinstance(e, HTTPException):\n raise\n _handle_unexpected_git_route_error(\"init_repository\", e)\n# [/DEF:init_repository:Function]\n\n\n# [DEF:get_repository_binding:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Return repository binding with provider metadata for selected dashboard.\n@router.get(\"/repositories/{dashboard_ref}\", response_model=RepositoryBindingSchema)\nasync def get_repository_binding(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_binding\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n db_repo = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if not db_repo:\n raise HTTPException(\n status_code=404, detail=\"Repository not initialized\"\n )\n config = _get_git_config_or_404(db, db_repo.config_id)\n return RepositoryBindingSchema(\n dashboard_id=db_repo.dashboard_id,\n config_id=db_repo.config_id,\n provider=config.provider,\n remote_url=db_repo.remote_url,\n local_path=db_repo.local_path,\n )\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_repository_binding\", e)\n# [/DEF:get_repository_binding:Function]\n\n\n# [DEF:delete_repository:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Delete local repository workspace and DB binding for selected dashboard.\n@router.delete(\"/repositories/{dashboard_ref}\")\nasync def delete_repository(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"delete_repository\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _gs.delete_repo(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"delete_repository\", e)\n# [/DEF:delete_repository:Function]\n\n\n# [DEF:get_branches:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: List all branches for a dashboard's repository.\n@router.get(\"/repositories/{dashboard_ref}/branches\", response_model=List[BranchSchema])\nasync def get_branches(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"get_branches\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n return _gs.list_branches(dashboard_id)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_branches\", e)\n# [/DEF:get_branches:Function]\n\n\n# [DEF:create_branch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Create a new branch in the dashboard's repository.\n@router.post(\"/repositories/{dashboard_ref}/branches\")\nasync def create_branch(\n dashboard_ref: str,\n branch_data: BranchCreate,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"create_branch\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.create_branch(\n dashboard_id, branch_data.name, branch_data.from_branch\n )\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"create_branch\", e)\n# [/DEF:create_branch:Function]\n\n\n# [DEF:checkout_branch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Switch the dashboard's repository to a specific branch.\n@router.post(\"/repositories/{dashboard_ref}/checkout\")\nasync def checkout_branch(\n dashboard_ref: str,\n checkout_data: BranchCheckout,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"checkout_branch\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _gs.checkout_branch(dashboard_id, checkout_data.name)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"checkout_branch\", e)\n# [/DEF:checkout_branch:Function]\n# [/DEF:GitRepoRoutes:Module]\n" + "body": "# [DEF:GitRepoRoutes:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: FastAPI endpoints for core Git repository operations (init, binding, branches, checkout).\n# @LAYER: API\n# @SEMANTICS: git, repository, routes, init, branches\n\nfrom typing import List, Optional\n\nfrom fastapi import Depends, HTTPException\nfrom sqlalchemy.orm import Session\n\nfrom src.core.database import get_db\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\n\nlog = MarkerLogger(\"GitRepoRoutes\")\nfrom src.dependencies import get_config_manager, get_current_user, has_permission\nfrom src.models.auth import User\nfrom src.models.git import GitRepository, GitServerConfig\n\nfrom src.api.routes.git_schemas import (\n BranchCheckout,\n BranchCreate,\n BranchSchema,\n RepoInitRequest,\n RepositoryBindingSchema,\n)\n\nfrom ._router import router\nfrom ._helpers import (\n _apply_git_identity_from_profile,\n _get_git_config_or_404,\n _handle_unexpected_git_route_error,\n)\nfrom ._deps import get_git_service\n\n\n# [DEF:init_repository:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/init\")\nasync def init_repository(\n dashboard_ref: str,\n init_data: RepoInitRequest,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"init_repository\"):\n from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref\n\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n repo_key = _resolve_repo_key_from_ref(\n dashboard_ref, dashboard_id, config_manager, env_id\n )\n config = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == init_data.config_id)\n .first()\n )\n if not config:\n raise HTTPException(status_code=404, detail=\"Git configuration not found\")\n\n try:\n log.reason(f\"Initializing repo for dashboard {dashboard_id}\")\n _gs.init_repo(\n dashboard_id,\n init_data.remote_url,\n config.pat,\n repo_key=repo_key,\n default_branch=config.default_branch,\n )\n\n repo_path = _gs._get_repo_path(dashboard_id, repo_key=repo_key)\n db_repo = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if not db_repo:\n db_repo = GitRepository(\n dashboard_id=dashboard_id,\n config_id=config.id,\n remote_url=init_data.remote_url,\n local_path=repo_path,\n current_branch=\"dev\",\n )\n db.add(db_repo)\n else:\n db_repo.config_id = config.id\n db_repo.remote_url = init_data.remote_url\n db_repo.local_path = repo_path\n db_repo.current_branch = \"dev\"\n\n db.commit()\n return {\"status\": \"success\", \"message\": \"Repository initialized\"}\n except Exception as e:\n db.rollback()\n log.explore(\"Failed to init repository\", error=str(e))\n if isinstance(e, HTTPException):\n raise\n _handle_unexpected_git_route_error(\"init_repository\", e)\n# [/DEF:init_repository:Function]\n\n\n# [DEF:get_repository_binding:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Return repository binding with provider metadata for selected dashboard.\n@router.get(\"/repositories/{dashboard_ref}\", response_model=RepositoryBindingSchema)\nasync def get_repository_binding(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n with belief_scope(\"get_repository_binding\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n db_repo = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if not db_repo:\n raise HTTPException(\n status_code=404, detail=\"Repository not initialized\"\n )\n config = _get_git_config_or_404(db, db_repo.config_id)\n return RepositoryBindingSchema(\n dashboard_id=db_repo.dashboard_id,\n config_id=db_repo.config_id,\n provider=config.provider,\n remote_url=db_repo.remote_url,\n local_path=db_repo.local_path,\n )\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_repository_binding\", e)\n# [/DEF:get_repository_binding:Function]\n\n\n# [DEF:delete_repository:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Delete local repository workspace and DB binding for selected dashboard.\n@router.delete(\"/repositories/{dashboard_ref}\")\nasync def delete_repository(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"delete_repository\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _gs.delete_repo(dashboard_id)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"delete_repository\", e)\n# [/DEF:delete_repository:Function]\n\n\n# [DEF:get_branches:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: List all branches for a dashboard's repository.\n@router.get(\"/repositories/{dashboard_ref}/branches\", response_model=List[BranchSchema])\nasync def get_branches(\n dashboard_ref: str,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"get_branches\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n return _gs.list_branches(dashboard_id)\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"get_branches\", e)\n# [/DEF:get_branches:Function]\n\n\n# [DEF:create_branch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Create a new branch in the dashboard's repository.\n@router.post(\"/repositories/{dashboard_ref}/branches\")\nasync def create_branch(\n dashboard_ref: str,\n branch_data: BranchCreate,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n current_user: User = Depends(get_current_user),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"create_branch\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _apply_git_identity_from_profile(dashboard_id, db, current_user)\n _gs.create_branch(\n dashboard_id, branch_data.name, branch_data.from_branch\n )\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"create_branch\", e)\n# [/DEF:create_branch:Function]\n\n\n# [DEF:checkout_branch:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Switch the dashboard's repository to a specific branch.\n@router.post(\"/repositories/{dashboard_ref}/checkout\")\nasync def checkout_branch(\n dashboard_ref: str,\n checkout_data: BranchCheckout,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"checkout_branch\"):\n from . import _resolve_dashboard_id_from_ref\n\n try:\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n _gs.checkout_branch(dashboard_id, checkout_data.name)\n return {\"status\": \"success\"}\n except HTTPException:\n raise\n except Exception as e:\n _handle_unexpected_git_route_error(\"checkout_branch\", e)\n# [/DEF:checkout_branch:Function]\n# [/DEF:GitRepoRoutes:Module]\n" }, { "contract_id": "init_repository", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_routes.py", - "start_line": 35, - "end_line": 112, + "start_line": 38, + "end_line": 108, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -19638,14 +19638,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:init_repository:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/init\")\nasync def init_repository(\n dashboard_ref: str,\n init_data: RepoInitRequest,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"init_repository\"):\n from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref\n\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n repo_key = _resolve_repo_key_from_ref(\n dashboard_ref, dashboard_id, config_manager, env_id\n )\n config = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == init_data.config_id)\n .first()\n )\n if not config:\n raise HTTPException(status_code=404, detail=\"Git configuration not found\")\n\n try:\n logger.info(\n f\"[init_repository][Action] Initializing repo for dashboard {dashboard_id}\"\n )\n _gs.init_repo(\n dashboard_id,\n init_data.remote_url,\n config.pat,\n repo_key=repo_key,\n default_branch=config.default_branch,\n )\n\n repo_path = _gs._get_repo_path(dashboard_id, repo_key=repo_key)\n db_repo = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if not db_repo:\n db_repo = GitRepository(\n dashboard_id=dashboard_id,\n config_id=config.id,\n remote_url=init_data.remote_url,\n local_path=repo_path,\n current_branch=\"dev\",\n )\n db.add(db_repo)\n else:\n db_repo.config_id = config.id\n db_repo.remote_url = init_data.remote_url\n db_repo.local_path = repo_path\n db_repo.current_branch = \"dev\"\n\n db.commit()\n logger.info(\n f\"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}\"\n )\n return {\"status\": \"success\", \"message\": \"Repository initialized\"}\n except Exception as e:\n db.rollback()\n logger.error(\n f\"[init_repository][Coherence:Failed] Failed to init repository: {e}\"\n )\n if isinstance(e, HTTPException):\n raise\n _handle_unexpected_git_route_error(\"init_repository\", e)\n# [/DEF:init_repository:Function]\n" + "body": "# [DEF:init_repository:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init.\n# @RELATION: CALLS -> [GitService]\n@router.post(\"/repositories/{dashboard_ref}/init\")\nasync def init_repository(\n dashboard_ref: str,\n init_data: RepoInitRequest,\n env_id: Optional[str] = None,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:git\", \"EXECUTE\")),\n):\n _gs = get_git_service()\n with belief_scope(\"init_repository\"):\n from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref\n\n dashboard_id = _resolve_dashboard_id_from_ref(\n dashboard_ref, config_manager, env_id\n )\n repo_key = _resolve_repo_key_from_ref(\n dashboard_ref, dashboard_id, config_manager, env_id\n )\n config = (\n db.query(GitServerConfig)\n .filter(GitServerConfig.id == init_data.config_id)\n .first()\n )\n if not config:\n raise HTTPException(status_code=404, detail=\"Git configuration not found\")\n\n try:\n log.reason(f\"Initializing repo for dashboard {dashboard_id}\")\n _gs.init_repo(\n dashboard_id,\n init_data.remote_url,\n config.pat,\n repo_key=repo_key,\n default_branch=config.default_branch,\n )\n\n repo_path = _gs._get_repo_path(dashboard_id, repo_key=repo_key)\n db_repo = (\n db.query(GitRepository)\n .filter(GitRepository.dashboard_id == dashboard_id)\n .first()\n )\n if not db_repo:\n db_repo = GitRepository(\n dashboard_id=dashboard_id,\n config_id=config.id,\n remote_url=init_data.remote_url,\n local_path=repo_path,\n current_branch=\"dev\",\n )\n db.add(db_repo)\n else:\n db_repo.config_id = config.id\n db_repo.remote_url = init_data.remote_url\n db_repo.local_path = repo_path\n db_repo.current_branch = \"dev\"\n\n db.commit()\n return {\"status\": \"success\", \"message\": \"Repository initialized\"}\n except Exception as e:\n db.rollback()\n log.explore(\"Failed to init repository\", error=str(e))\n if isinstance(e, HTTPException):\n raise\n _handle_unexpected_git_route_error(\"init_repository\", e)\n# [/DEF:init_repository:Function]\n" }, { "contract_id": "get_repository_binding", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_routes.py", - "start_line": 115, - "end_line": 154, + "start_line": 111, + "end_line": 150, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19661,8 +19661,8 @@ "contract_id": "delete_repository", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_routes.py", - "start_line": 157, - "end_line": 181, + "start_line": 153, + "end_line": 177, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19678,8 +19678,8 @@ "contract_id": "get_branches", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_routes.py", - "start_line": 184, - "end_line": 207, + "start_line": 180, + "end_line": 203, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19695,8 +19695,8 @@ "contract_id": "create_branch", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_routes.py", - "start_line": 210, - "end_line": 240, + "start_line": 206, + "end_line": 236, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -19712,8 +19712,8 @@ "contract_id": "checkout_branch", "contract_type": "Function", "file_path": "backend/src/api/routes/git/_repo_routes.py", - "start_line": 243, - "end_line": 268, + "start_line": 239, + "end_line": 264, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -20419,7 +20419,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/llm.py", "start_line": 1, - "end_line": 488, + "end_line": 481, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -20475,14 +20475,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:LlmRoutes:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: api, routes, llm\n# @PURPOSE: API routes for LLM provider configuration and management.\n# @LAYER: UI (API)\n# @RELATION: DEPENDS_ON -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n# @RELATION: DEPENDS_ON -> [get_current_user]\n# @RELATION: DEPENDS_ON -> [get_db]\n\nfrom fastapi import APIRouter, Depends, HTTPException, status\nfrom typing import List, Optional\nimport httpx\nfrom ...core.logger import logger\nfrom ...schemas.auth import User\nfrom ...dependencies import get_current_user as get_current_active_user\nfrom ...plugins.llm_analysis.models import (\n LLMProviderConfig,\n LLMProviderType,\n FetchModelsRequest,\n FetchModelsResponse,\n)\nfrom ...services.llm_provider import LLMProviderService\nfrom ...core.database import get_db\nfrom sqlalchemy.orm import Session\n\n# [DEF:router:Global]\n# @PURPOSE: APIRouter instance for LLM routes.\nrouter = APIRouter(tags=[\"LLM\"])\n# [/DEF:router:Global]\n\n\n# [DEF:_is_valid_runtime_api_key:Function]\n# @PURPOSE: Validate decrypted runtime API key presence/shape.\n# @PRE: value can be None.\n# @POST: Returns True only for non-placeholder key.\n# @RELATION: BINDS_TO -> [LlmRoutes]\ndef _is_valid_runtime_api_key(value: Optional[str]) -> bool:\n key = (value or \"\").strip()\n if not key:\n return False\n if key in {\"********\", \"EMPTY_OR_NONE\"}:\n return False\n return len(key) >= 16\n\n\n# [/DEF:_is_valid_runtime_api_key:Function]\n\n\n# [DEF:_build_provider_auth_headers:Function]\n# @PURPOSE: Build auth headers for provider model listing based on provider type.\n# @PRE: provider_type is a valid LLMProviderType.\n# @POST: Returns dict of HTTP headers for the provider's model list API.\ndef _build_provider_auth_headers(api_key: Optional[str], provider_type: LLMProviderType) -> dict:\n headers = {}\n if not api_key:\n return headers\n if provider_type == LLMProviderType.KILO:\n headers[\"Authentication\"] = f\"Bearer {api_key}\"\n headers[\"X-API-Key\"] = api_key\n else:\n # OpenAI, OpenRouter — standard Bearer token\n headers[\"Authorization\"] = f\"Bearer {api_key}\"\n return headers\n\n\n# [/DEF:_build_provider_auth_headers:Function]\n\n\n# [DEF:get_providers:Function]\n# @PURPOSE: Retrieve all LLM provider configurations.\n# @PRE: User is authenticated.\n# @POST: Returns list of LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.get(\"/providers\", response_model=List[LLMProviderConfig])\nasync def get_providers(\n current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)\n):\n \"\"\"\n Get all LLM provider configurations.\n \"\"\"\n logger.info(\n f\"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}\"\n )\n service = LLMProviderService(db)\n providers = service.get_all_providers()\n return [\n LLMProviderConfig(\n id=p.id,\n provider_type=LLMProviderType(p.provider_type),\n name=p.name,\n base_url=p.base_url,\n api_key=\"********\",\n default_model=p.default_model,\n is_active=p.is_active,\n )\n for p in providers\n ]\n\n\n# [/DEF:get_providers:Function]\n\n\n# [DEF:get_llm_status:Function]\n# @PURPOSE: Returns whether LLM runtime is configured for dashboard validation.\n# @PRE: User is authenticated.\n# @POST: configured=true only when an active provider with valid decrypted key exists.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: CALLS -> [_is_valid_runtime_api_key]\n@router.get(\"/status\")\nasync def get_llm_status(\n current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)\n):\n service = LLMProviderService(db)\n providers = service.get_all_providers()\n active_provider = next((p for p in providers if p.is_active), None)\n\n if not providers:\n return {\n \"configured\": False,\n \"reason\": \"no_providers_configured\",\n \"provider_count\": 0,\n \"active_provider_count\": 0,\n }\n\n if not active_provider:\n return {\n \"configured\": False,\n \"reason\": \"no_active_provider\",\n \"provider_count\": len(providers),\n \"active_provider_count\": 0,\n \"providers\": [\n {\n \"id\": provider.id,\n \"name\": provider.name,\n \"provider_type\": provider.provider_type,\n \"is_active\": bool(provider.is_active),\n }\n for provider in providers\n ],\n }\n\n api_key = service.get_decrypted_api_key(active_provider.id)\n if not _is_valid_runtime_api_key(api_key):\n return {\n \"configured\": False,\n \"reason\": \"invalid_api_key\",\n \"provider_count\": len(providers),\n \"active_provider_count\": len(\n [provider for provider in providers if provider.is_active]\n ),\n \"provider_id\": active_provider.id,\n \"provider_name\": active_provider.name,\n \"provider_type\": active_provider.provider_type,\n \"default_model\": active_provider.default_model,\n }\n\n return {\n \"configured\": True,\n \"reason\": \"ok\",\n \"provider_count\": len(providers),\n \"active_provider_count\": len(\n [provider for provider in providers if provider.is_active]\n ),\n \"provider_id\": active_provider.id,\n \"provider_name\": active_provider.name,\n \"provider_type\": active_provider.provider_type,\n \"default_model\": active_provider.default_model,\n }\n\n\n# [/DEF:get_llm_status:Function]\n\n\n# [DEF:create_provider:Function]\n# @PURPOSE: Create a new LLM provider configuration.\n# @PRE: User is authenticated and has admin permissions.\n# @POST: Returns the created LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.post(\n \"/providers\", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED\n)\nasync def create_provider(\n config: LLMProviderConfig,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Create a new LLM provider configuration.\n \"\"\"\n service = LLMProviderService(db)\n provider = service.create_provider(config)\n return LLMProviderConfig(\n id=provider.id,\n provider_type=LLMProviderType(provider.provider_type),\n name=provider.name,\n base_url=provider.base_url,\n api_key=\"********\",\n default_model=provider.default_model,\n is_active=provider.is_active,\n )\n\n\n# [/DEF:create_provider:Function]\n\n\n# [DEF:update_provider:Function]\n# @PURPOSE: Update an existing LLM provider configuration.\n# @PRE: User is authenticated and has admin permissions.\n# @POST: Returns the updated LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.put(\"/providers/{provider_id}\", response_model=LLMProviderConfig)\nasync def update_provider(\n provider_id: str,\n config: LLMProviderConfig,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Update an existing LLM provider configuration.\n \"\"\"\n service = LLMProviderService(db)\n provider = service.update_provider(provider_id, config)\n if not provider:\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n\n return LLMProviderConfig(\n id=provider.id,\n provider_type=LLMProviderType(provider.provider_type),\n name=provider.name,\n base_url=provider.base_url,\n api_key=\"********\",\n default_model=provider.default_model,\n is_active=provider.is_active,\n )\n\n\n# [/DEF:update_provider:Function]\n\n\n# [DEF:delete_provider:Function]\n# @PURPOSE: Delete an LLM provider configuration.\n# @PRE: User is authenticated and has admin permissions.\n# @POST: Returns success status.\n# @RELATION: CALLS -> [LLMProviderService]\n@router.delete(\"/providers/{provider_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_provider(\n provider_id: str,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Delete an LLM provider configuration.\n \"\"\"\n service = LLMProviderService(db)\n if not service.delete_provider(provider_id):\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n return\n\n\n# [/DEF:delete_provider:Function]\n\n\n# [DEF:fetch_models_for_provider:Function]\n# @PURPOSE: Fetch available models from a provider's API endpoint.\n# @PRE: valid base_url and optional api_key/provider_id.\n# @POST: Returns list of model IDs sorted alphabetically.\n# @SIDE_EFFECT: Makes HTTP GET to multiple possible model endpoints.\n@router.post(\"/providers/fetch-models\", response_model=FetchModelsResponse)\nasync def fetch_models_for_provider(\n request: FetchModelsRequest,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Fetch available models from a provider's models endpoint.\n\n Resolves API key in priority order:\n 1. Direct api_key from request body\n 2. Decrypted api_key from DB via provider_id (for existing providers)\n 3. No auth (for local providers without auth)\n\n Tries multiple paths in order:\n 1. {base_url}/models (OpenAI-compatible, stripped of /v1)\n 2. {base_url}/v1/models (OpenAI-compatible, with /v1)\n 3. {base_url}/api/tags (Ollama format)\n Returns empty list if none respond — user can enter model name manually.\n \"\"\"\n logger.info(\n f\"[llm_routes][fetch_models] Fetching models for {request.base_url}\"\n )\n\n # Resolve API key: prefer direct, fall back to stored key via provider_id\n api_key = request.api_key\n if not api_key and request.provider_id:\n try:\n svc = LLMProviderService(db)\n api_key = svc.get_decrypted_api_key(request.provider_id)\n except Exception:\n logger.warning(\n f\"[llm_routes][fetch_models] Could not resolve API key for provider {request.provider_id}\"\n )\n\n base = request.base_url.rstrip(\"/\")\n\n # Collect possible model list endpoints\n candidates = []\n\n # If base ends with /v1, try both /models (after stripping /v1) and /v1/models\n if base.endswith(\"/v1\"):\n candidates.append(base[:-3] + \"/models\") # base without /v1 + /models\n candidates.append(base + \"/models\") # base with /v1 + /models (e.g. /v1/models)\n else:\n candidates.append(base + \"/models\") # /models\n candidates.append(base + \"/v1/models\") # /v1/models\n\n candidates.append(base + \"/api/tags\") # Ollama format\n\n headers = _build_provider_auth_headers(api_key, request.provider_type)\n last_error = None\n\n try:\n async with httpx.AsyncClient(timeout=10.0) as client:\n for url in candidates:\n try:\n response = await client.get(url, headers=headers)\n\n if response.status_code != 200:\n if response.status_code != 404:\n last_error = f\"HTTP {response.status_code} from {url}: {response.text[:200]}\"\n continue\n\n body = response.json()\n raw = body.get(\"data\") if isinstance(body, dict) else body\n\n # Try OpenAI format: [{\"id\": \"model-name\", ...}, ...] or list of strings\n if isinstance(raw, list):\n models = sorted(\n m.get(\"id\", str(m)) if isinstance(m, dict) else str(m)\n for m in raw\n )\n if models:\n logger.info(\n f\"[llm_routes][fetch_models] Found {len(models)} models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n # Try Ollama format: {\"models\": [{\"name\": \"model-name\", ...}, ...]}\n if isinstance(body, dict):\n models_list = body.get(\"models\", [])\n if isinstance(models_list, list):\n models = sorted(\n m.get(\"name\", str(m)) if isinstance(m, dict) else str(m)\n for m in models_list\n )\n if models:\n logger.info(\n f\"[llm_routes][fetch_models] Found {len(models)} Ollama models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n except httpx.TimeoutException:\n last_error = f\"Timeout at {url}\"\n except httpx.ConnectError:\n last_error = f\"Cannot connect at {url}\"\n except Exception as e:\n last_error = f\"Error at {url}: {str(e)[:100]}\"\n\n # All endpoints failed — return empty list with no error (user can type manually)\n logger.warning(\n f\"[llm_routes][fetch_models] No model endpoint responded for {request.base_url}. \"\n f\"Last error: {last_error}\"\n )\n return FetchModelsResponse(models=[], total=0)\n\n except Exception as e:\n logger.error(\n f\"[llm_routes][fetch_models] Unexpected error for {request.base_url}: {e}\"\n )\n return FetchModelsResponse(models=[], total=0)\n\n\n# [/DEF:fetch_models_for_provider:Function]\n\n\n# [DEF:test_connection:Function]\n# @PURPOSE: Test connection to an LLM provider.\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMClient]\n@router.post(\"/providers/{provider_id}/test\")\nasync def test_connection(\n provider_id: str,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n logger.info(\n f\"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}\"\n )\n \"\"\"\n Test connection to an LLM provider.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n service = LLMProviderService(db)\n db_provider = service.get_provider(provider_id)\n if not db_provider:\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n\n api_key = service.get_decrypted_api_key(provider_id)\n\n # Check if API key was successfully decrypted\n if not api_key:\n logger.error(\n f\"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}\"\n )\n raise HTTPException(\n status_code=500,\n detail=\"Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.\",\n )\n\n client = LLMClient(\n provider_type=LLMProviderType(db_provider.provider_type),\n api_key=api_key,\n base_url=db_provider.base_url,\n default_model=db_provider.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_connection:Function]\n\n\n# [DEF:test_provider_config:Function]\n# @PURPOSE: Test connection with a provided configuration (not yet saved).\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: DEPENDS_ON -> [LLMClient]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.post(\"/providers/test\")\nasync def test_provider_config(\n config: LLMProviderConfig, current_user: User = Depends(get_current_active_user)\n):\n \"\"\"\n Test connection with a provided configuration.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n logger.info(\n f\"[llm_routes][test_provider_config][Action] Testing config for {config.name}\"\n )\n\n # Check if API key is provided\n if not config.api_key or config.api_key == \"********\":\n raise HTTPException(\n status_code=400, detail=\"API key is required for testing connection\"\n )\n\n client = LLMClient(\n provider_type=config.provider_type,\n api_key=config.api_key,\n base_url=config.base_url,\n default_model=config.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_provider_config:Function]\n\n# [/DEF:LlmRoutes:Module]\n" + "body": "# [DEF:LlmRoutes:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: api, routes, llm\n# @PURPOSE: API routes for LLM provider configuration and management.\n# @LAYER: UI (API)\n# @RELATION: DEPENDS_ON -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n# @RELATION: DEPENDS_ON -> [get_current_user]\n# @RELATION: DEPENDS_ON -> [get_db]\n\nfrom fastapi import APIRouter, Depends, HTTPException, status\nfrom typing import List, Optional\nimport httpx\nfrom ...core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"LlmRoutes\")\nfrom ...schemas.auth import User\nfrom ...dependencies import get_current_user as get_current_active_user\nfrom ...plugins.llm_analysis.models import (\n LLMProviderConfig,\n LLMProviderType,\n FetchModelsRequest,\n FetchModelsResponse,\n)\nfrom ...services.llm_provider import LLMProviderService\nfrom ...core.database import get_db\nfrom sqlalchemy.orm import Session\n\n# [DEF:router:Global]\n# @PURPOSE: APIRouter instance for LLM routes.\nrouter = APIRouter(tags=[\"LLM\"])\n# [/DEF:router:Global]\n\n\n# [DEF:_is_valid_runtime_api_key:Function]\n# @PURPOSE: Validate decrypted runtime API key presence/shape.\n# @PRE: value can be None.\n# @POST: Returns True only for non-placeholder key.\n# @RELATION: BINDS_TO -> [LlmRoutes]\ndef _is_valid_runtime_api_key(value: Optional[str]) -> bool:\n key = (value or \"\").strip()\n if not key:\n return False\n if key in {\"********\", \"EMPTY_OR_NONE\"}:\n return False\n return len(key) >= 16\n\n\n# [/DEF:_is_valid_runtime_api_key:Function]\n\n\n# [DEF:_build_provider_auth_headers:Function]\n# @PURPOSE: Build auth headers for provider model listing based on provider type.\n# @PRE: provider_type is a valid LLMProviderType.\n# @POST: Returns dict of HTTP headers for the provider's model list API.\ndef _build_provider_auth_headers(api_key: Optional[str], provider_type: LLMProviderType) -> dict:\n headers = {}\n if not api_key:\n return headers\n if provider_type == LLMProviderType.KILO:\n headers[\"Authentication\"] = f\"Bearer {api_key}\"\n headers[\"X-API-Key\"] = api_key\n else:\n # OpenAI, OpenRouter — standard Bearer token\n headers[\"Authorization\"] = f\"Bearer {api_key}\"\n return headers\n\n\n# [/DEF:_build_provider_auth_headers:Function]\n\n\n# [DEF:get_providers:Function]\n# @PURPOSE: Retrieve all LLM provider configurations.\n# @PRE: User is authenticated.\n# @POST: Returns list of LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.get(\"/providers\", response_model=List[LLMProviderConfig])\nasync def get_providers(\n current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)\n):\n \"\"\"\n Get all LLM provider configurations.\n \"\"\"\n log.reason(\n f\"Fetching providers for user: {current_user.username}\"\n )\n service = LLMProviderService(db)\n providers = service.get_all_providers()\n return [\n LLMProviderConfig(\n id=p.id,\n provider_type=LLMProviderType(p.provider_type),\n name=p.name,\n base_url=p.base_url,\n api_key=\"********\",\n default_model=p.default_model,\n is_active=p.is_active,\n )\n for p in providers\n ]\n\n\n# [/DEF:get_providers:Function]\n\n\n# [DEF:get_llm_status:Function]\n# @PURPOSE: Returns whether LLM runtime is configured for dashboard validation.\n# @PRE: User is authenticated.\n# @POST: configured=true only when an active provider with valid decrypted key exists.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: CALLS -> [_is_valid_runtime_api_key]\n@router.get(\"/status\")\nasync def get_llm_status(\n current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)\n):\n service = LLMProviderService(db)\n providers = service.get_all_providers()\n active_provider = next((p for p in providers if p.is_active), None)\n\n if not providers:\n return {\n \"configured\": False,\n \"reason\": \"no_providers_configured\",\n \"provider_count\": 0,\n \"active_provider_count\": 0,\n }\n\n if not active_provider:\n return {\n \"configured\": False,\n \"reason\": \"no_active_provider\",\n \"provider_count\": len(providers),\n \"active_provider_count\": 0,\n \"providers\": [\n {\n \"id\": provider.id,\n \"name\": provider.name,\n \"provider_type\": provider.provider_type,\n \"is_active\": bool(provider.is_active),\n }\n for provider in providers\n ],\n }\n\n api_key = service.get_decrypted_api_key(active_provider.id)\n if not _is_valid_runtime_api_key(api_key):\n return {\n \"configured\": False,\n \"reason\": \"invalid_api_key\",\n \"provider_count\": len(providers),\n \"active_provider_count\": len(\n [provider for provider in providers if provider.is_active]\n ),\n \"provider_id\": active_provider.id,\n \"provider_name\": active_provider.name,\n \"provider_type\": active_provider.provider_type,\n \"default_model\": active_provider.default_model,\n }\n\n return {\n \"configured\": True,\n \"reason\": \"ok\",\n \"provider_count\": len(providers),\n \"active_provider_count\": len(\n [provider for provider in providers if provider.is_active]\n ),\n \"provider_id\": active_provider.id,\n \"provider_name\": active_provider.name,\n \"provider_type\": active_provider.provider_type,\n \"default_model\": active_provider.default_model,\n }\n\n\n# [/DEF:get_llm_status:Function]\n\n\n# [DEF:create_provider:Function]\n# @PURPOSE: Create a new LLM provider configuration.\n# @PRE: User is authenticated and has admin permissions.\n# @POST: Returns the created LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.post(\n \"/providers\", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED\n)\nasync def create_provider(\n config: LLMProviderConfig,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Create a new LLM provider configuration.\n \"\"\"\n service = LLMProviderService(db)\n provider = service.create_provider(config)\n return LLMProviderConfig(\n id=provider.id,\n provider_type=LLMProviderType(provider.provider_type),\n name=provider.name,\n base_url=provider.base_url,\n api_key=\"********\",\n default_model=provider.default_model,\n is_active=provider.is_active,\n )\n\n\n# [/DEF:create_provider:Function]\n\n\n# [DEF:update_provider:Function]\n# @PURPOSE: Update an existing LLM provider configuration.\n# @PRE: User is authenticated and has admin permissions.\n# @POST: Returns the updated LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.put(\"/providers/{provider_id}\", response_model=LLMProviderConfig)\nasync def update_provider(\n provider_id: str,\n config: LLMProviderConfig,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Update an existing LLM provider configuration.\n \"\"\"\n service = LLMProviderService(db)\n provider = service.update_provider(provider_id, config)\n if not provider:\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n\n return LLMProviderConfig(\n id=provider.id,\n provider_type=LLMProviderType(provider.provider_type),\n name=provider.name,\n base_url=provider.base_url,\n api_key=\"********\",\n default_model=provider.default_model,\n is_active=provider.is_active,\n )\n\n\n# [/DEF:update_provider:Function]\n\n\n# [DEF:delete_provider:Function]\n# @PURPOSE: Delete an LLM provider configuration.\n# @PRE: User is authenticated and has admin permissions.\n# @POST: Returns success status.\n# @RELATION: CALLS -> [LLMProviderService]\n@router.delete(\"/providers/{provider_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_provider(\n provider_id: str,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Delete an LLM provider configuration.\n \"\"\"\n service = LLMProviderService(db)\n if not service.delete_provider(provider_id):\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n return\n\n\n# [/DEF:delete_provider:Function]\n\n\n# [DEF:fetch_models_for_provider:Function]\n# @PURPOSE: Fetch available models from a provider's API endpoint.\n# @PRE: valid base_url and optional api_key/provider_id.\n# @POST: Returns list of model IDs sorted alphabetically.\n# @SIDE_EFFECT: Makes HTTP GET to multiple possible model endpoints.\n@router.post(\"/providers/fetch-models\", response_model=FetchModelsResponse)\nasync def fetch_models_for_provider(\n request: FetchModelsRequest,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Fetch available models from a provider's models endpoint.\n\n Resolves API key in priority order:\n 1. Direct api_key from request body\n 2. Decrypted api_key from DB via provider_id (for existing providers)\n 3. No auth (for local providers without auth)\n\n Tries multiple paths in order:\n 1. {base_url}/models (OpenAI-compatible, stripped of /v1)\n 2. {base_url}/v1/models (OpenAI-compatible, with /v1)\n 3. {base_url}/api/tags (Ollama format)\n Returns empty list if none respond — user can enter model name manually.\n \"\"\"\n log.reason(\n f\"Fetching models for {request.base_url}\"\n )\n\n # Resolve API key: prefer direct, fall back to stored key via provider_id\n api_key = request.api_key\n if not api_key and request.provider_id:\n try:\n svc = LLMProviderService(db)\n api_key = svc.get_decrypted_api_key(request.provider_id)\n except Exception:\n log.explore(f\"Could not resolve API key for provider {request.provider_id}\", error=\"API key resolution failed\")\n\n base = request.base_url.rstrip(\"/\")\n\n # Collect possible model list endpoints\n candidates = []\n\n # If base ends with /v1, try both /models (after stripping /v1) and /v1/models\n if base.endswith(\"/v1\"):\n candidates.append(base[:-3] + \"/models\") # base without /v1 + /models\n candidates.append(base + \"/models\") # base with /v1 + /models (e.g. /v1/models)\n else:\n candidates.append(base + \"/models\") # /models\n candidates.append(base + \"/v1/models\") # /v1/models\n\n candidates.append(base + \"/api/tags\") # Ollama format\n\n headers = _build_provider_auth_headers(api_key, request.provider_type)\n last_error = None\n\n try:\n async with httpx.AsyncClient(timeout=10.0) as client:\n for url in candidates:\n try:\n response = await client.get(url, headers=headers)\n\n if response.status_code != 200:\n if response.status_code != 404:\n last_error = f\"HTTP {response.status_code} from {url}: {response.text[:200]}\"\n continue\n\n body = response.json()\n raw = body.get(\"data\") if isinstance(body, dict) else body\n\n # Try OpenAI format: [{\"id\": \"model-name\", ...}, ...] or list of strings\n if isinstance(raw, list):\n models = sorted(\n m.get(\"id\", str(m)) if isinstance(m, dict) else str(m)\n for m in raw\n )\n if models:\n log.reason(\n f\"Found {len(models)} models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n # Try Ollama format: {\"models\": [{\"name\": \"model-name\", ...}, ...]}\n if isinstance(body, dict):\n models_list = body.get(\"models\", [])\n if isinstance(models_list, list):\n models = sorted(\n m.get(\"name\", str(m)) if isinstance(m, dict) else str(m)\n for m in models_list\n )\n if models:\n log.reason(\n f\"Found {len(models)} Ollama models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n except httpx.TimeoutException:\n last_error = f\"Timeout at {url}\"\n except httpx.ConnectError:\n last_error = f\"Cannot connect at {url}\"\n except Exception as e:\n last_error = f\"Error at {url}: {str(e)[:100]}\"\n\n # All endpoints failed — return empty list with no error (user can type manually)\n log.explore(f\"No model endpoint responded for {request.base_url}. Last error: {last_error}\", error=f\"No model endpoint responded for {request.base_url}\")\n return FetchModelsResponse(models=[], total=0)\n\n except Exception as e:\n log.explore(f\"Unexpected error for {request.base_url}: {e}\", error=str(e))\n return FetchModelsResponse(models=[], total=0)\n\n\n# [/DEF:fetch_models_for_provider:Function]\n\n\n# [DEF:test_connection:Function]\n# @PURPOSE: Test connection to an LLM provider.\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMClient]\n@router.post(\"/providers/{provider_id}/test\")\nasync def test_connection(\n provider_id: str,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n log.reason(\n f\"Testing connection for provider_id: {provider_id}\"\n )\n \"\"\"\n Test connection to an LLM provider.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n service = LLMProviderService(db)\n db_provider = service.get_provider(provider_id)\n if not db_provider:\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n\n api_key = service.get_decrypted_api_key(provider_id)\n\n # Check if API key was successfully decrypted\n if not api_key:\n log.explore(f\"Failed to decrypt API key for provider {provider_id}\", error=\"API key decryption failed\")\n raise HTTPException(\n status_code=500,\n detail=\"Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.\",\n )\n\n client = LLMClient(\n provider_type=LLMProviderType(db_provider.provider_type),\n api_key=api_key,\n base_url=db_provider.base_url,\n default_model=db_provider.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_connection:Function]\n\n\n# [DEF:test_provider_config:Function]\n# @PURPOSE: Test connection with a provided configuration (not yet saved).\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: DEPENDS_ON -> [LLMClient]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.post(\"/providers/test\")\nasync def test_provider_config(\n config: LLMProviderConfig, current_user: User = Depends(get_current_active_user)\n):\n \"\"\"\n Test connection with a provided configuration.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n log.reason(\n f\"Testing config for {config.name}\"\n )\n\n # Check if API key is provided\n if not config.api_key or config.api_key == \"********\":\n raise HTTPException(\n status_code=400, detail=\"API key is required for testing connection\"\n )\n\n client = LLMClient(\n provider_type=config.provider_type,\n api_key=config.api_key,\n base_url=config.base_url,\n default_model=config.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_provider_config:Function]\n\n# [/DEF:LlmRoutes:Module]\n" }, { "contract_id": "router", "contract_type": "Global", "file_path": "backend/src/api/routes/llm.py", - "start_line": 27, - "end_line": 30, + "start_line": 29, + "end_line": 32, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20523,8 +20523,8 @@ "contract_id": "_is_valid_runtime_api_key", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 33, - "end_line": 47, + "start_line": 35, + "end_line": 49, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20576,8 +20576,8 @@ "contract_id": "_build_provider_auth_headers", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 50, - "end_line": 67, + "start_line": 52, + "end_line": 69, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20613,8 +20613,8 @@ "contract_id": "get_providers", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 70, - "end_line": 102, + "start_line": 72, + "end_line": 104, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20666,14 +20666,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_providers:Function]\n# @PURPOSE: Retrieve all LLM provider configurations.\n# @PRE: User is authenticated.\n# @POST: Returns list of LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.get(\"/providers\", response_model=List[LLMProviderConfig])\nasync def get_providers(\n current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)\n):\n \"\"\"\n Get all LLM provider configurations.\n \"\"\"\n logger.info(\n f\"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}\"\n )\n service = LLMProviderService(db)\n providers = service.get_all_providers()\n return [\n LLMProviderConfig(\n id=p.id,\n provider_type=LLMProviderType(p.provider_type),\n name=p.name,\n base_url=p.base_url,\n api_key=\"********\",\n default_model=p.default_model,\n is_active=p.is_active,\n )\n for p in providers\n ]\n\n\n# [/DEF:get_providers:Function]\n" + "body": "# [DEF:get_providers:Function]\n# @PURPOSE: Retrieve all LLM provider configurations.\n# @PRE: User is authenticated.\n# @POST: Returns list of LLMProviderConfig.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.get(\"/providers\", response_model=List[LLMProviderConfig])\nasync def get_providers(\n current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)\n):\n \"\"\"\n Get all LLM provider configurations.\n \"\"\"\n log.reason(\n f\"Fetching providers for user: {current_user.username}\"\n )\n service = LLMProviderService(db)\n providers = service.get_all_providers()\n return [\n LLMProviderConfig(\n id=p.id,\n provider_type=LLMProviderType(p.provider_type),\n name=p.name,\n base_url=p.base_url,\n api_key=\"********\",\n default_model=p.default_model,\n is_active=p.is_active,\n )\n for p in providers\n ]\n\n\n# [/DEF:get_providers:Function]\n" }, { "contract_id": "get_llm_status", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 105, - "end_line": 173, + "start_line": 107, + "end_line": 175, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20731,8 +20731,8 @@ "contract_id": "create_provider", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 176, - "end_line": 206, + "start_line": 178, + "end_line": 208, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20790,8 +20790,8 @@ "contract_id": "update_provider", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 209, - "end_line": 241, + "start_line": 211, + "end_line": 243, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20849,8 +20849,8 @@ "contract_id": "delete_provider", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 244, - "end_line": 264, + "start_line": 246, + "end_line": 266, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20902,8 +20902,8 @@ "contract_id": "fetch_models_for_provider", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 267, - "end_line": 390, + "start_line": 269, + "end_line": 385, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -20943,14 +20943,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:fetch_models_for_provider:Function]\n# @PURPOSE: Fetch available models from a provider's API endpoint.\n# @PRE: valid base_url and optional api_key/provider_id.\n# @POST: Returns list of model IDs sorted alphabetically.\n# @SIDE_EFFECT: Makes HTTP GET to multiple possible model endpoints.\n@router.post(\"/providers/fetch-models\", response_model=FetchModelsResponse)\nasync def fetch_models_for_provider(\n request: FetchModelsRequest,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Fetch available models from a provider's models endpoint.\n\n Resolves API key in priority order:\n 1. Direct api_key from request body\n 2. Decrypted api_key from DB via provider_id (for existing providers)\n 3. No auth (for local providers without auth)\n\n Tries multiple paths in order:\n 1. {base_url}/models (OpenAI-compatible, stripped of /v1)\n 2. {base_url}/v1/models (OpenAI-compatible, with /v1)\n 3. {base_url}/api/tags (Ollama format)\n Returns empty list if none respond — user can enter model name manually.\n \"\"\"\n logger.info(\n f\"[llm_routes][fetch_models] Fetching models for {request.base_url}\"\n )\n\n # Resolve API key: prefer direct, fall back to stored key via provider_id\n api_key = request.api_key\n if not api_key and request.provider_id:\n try:\n svc = LLMProviderService(db)\n api_key = svc.get_decrypted_api_key(request.provider_id)\n except Exception:\n logger.warning(\n f\"[llm_routes][fetch_models] Could not resolve API key for provider {request.provider_id}\"\n )\n\n base = request.base_url.rstrip(\"/\")\n\n # Collect possible model list endpoints\n candidates = []\n\n # If base ends with /v1, try both /models (after stripping /v1) and /v1/models\n if base.endswith(\"/v1\"):\n candidates.append(base[:-3] + \"/models\") # base without /v1 + /models\n candidates.append(base + \"/models\") # base with /v1 + /models (e.g. /v1/models)\n else:\n candidates.append(base + \"/models\") # /models\n candidates.append(base + \"/v1/models\") # /v1/models\n\n candidates.append(base + \"/api/tags\") # Ollama format\n\n headers = _build_provider_auth_headers(api_key, request.provider_type)\n last_error = None\n\n try:\n async with httpx.AsyncClient(timeout=10.0) as client:\n for url in candidates:\n try:\n response = await client.get(url, headers=headers)\n\n if response.status_code != 200:\n if response.status_code != 404:\n last_error = f\"HTTP {response.status_code} from {url}: {response.text[:200]}\"\n continue\n\n body = response.json()\n raw = body.get(\"data\") if isinstance(body, dict) else body\n\n # Try OpenAI format: [{\"id\": \"model-name\", ...}, ...] or list of strings\n if isinstance(raw, list):\n models = sorted(\n m.get(\"id\", str(m)) if isinstance(m, dict) else str(m)\n for m in raw\n )\n if models:\n logger.info(\n f\"[llm_routes][fetch_models] Found {len(models)} models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n # Try Ollama format: {\"models\": [{\"name\": \"model-name\", ...}, ...]}\n if isinstance(body, dict):\n models_list = body.get(\"models\", [])\n if isinstance(models_list, list):\n models = sorted(\n m.get(\"name\", str(m)) if isinstance(m, dict) else str(m)\n for m in models_list\n )\n if models:\n logger.info(\n f\"[llm_routes][fetch_models] Found {len(models)} Ollama models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n except httpx.TimeoutException:\n last_error = f\"Timeout at {url}\"\n except httpx.ConnectError:\n last_error = f\"Cannot connect at {url}\"\n except Exception as e:\n last_error = f\"Error at {url}: {str(e)[:100]}\"\n\n # All endpoints failed — return empty list with no error (user can type manually)\n logger.warning(\n f\"[llm_routes][fetch_models] No model endpoint responded for {request.base_url}. \"\n f\"Last error: {last_error}\"\n )\n return FetchModelsResponse(models=[], total=0)\n\n except Exception as e:\n logger.error(\n f\"[llm_routes][fetch_models] Unexpected error for {request.base_url}: {e}\"\n )\n return FetchModelsResponse(models=[], total=0)\n\n\n# [/DEF:fetch_models_for_provider:Function]\n" + "body": "# [DEF:fetch_models_for_provider:Function]\n# @PURPOSE: Fetch available models from a provider's API endpoint.\n# @PRE: valid base_url and optional api_key/provider_id.\n# @POST: Returns list of model IDs sorted alphabetically.\n# @SIDE_EFFECT: Makes HTTP GET to multiple possible model endpoints.\n@router.post(\"/providers/fetch-models\", response_model=FetchModelsResponse)\nasync def fetch_models_for_provider(\n request: FetchModelsRequest,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n \"\"\"\n Fetch available models from a provider's models endpoint.\n\n Resolves API key in priority order:\n 1. Direct api_key from request body\n 2. Decrypted api_key from DB via provider_id (for existing providers)\n 3. No auth (for local providers without auth)\n\n Tries multiple paths in order:\n 1. {base_url}/models (OpenAI-compatible, stripped of /v1)\n 2. {base_url}/v1/models (OpenAI-compatible, with /v1)\n 3. {base_url}/api/tags (Ollama format)\n Returns empty list if none respond — user can enter model name manually.\n \"\"\"\n log.reason(\n f\"Fetching models for {request.base_url}\"\n )\n\n # Resolve API key: prefer direct, fall back to stored key via provider_id\n api_key = request.api_key\n if not api_key and request.provider_id:\n try:\n svc = LLMProviderService(db)\n api_key = svc.get_decrypted_api_key(request.provider_id)\n except Exception:\n log.explore(f\"Could not resolve API key for provider {request.provider_id}\", error=\"API key resolution failed\")\n\n base = request.base_url.rstrip(\"/\")\n\n # Collect possible model list endpoints\n candidates = []\n\n # If base ends with /v1, try both /models (after stripping /v1) and /v1/models\n if base.endswith(\"/v1\"):\n candidates.append(base[:-3] + \"/models\") # base without /v1 + /models\n candidates.append(base + \"/models\") # base with /v1 + /models (e.g. /v1/models)\n else:\n candidates.append(base + \"/models\") # /models\n candidates.append(base + \"/v1/models\") # /v1/models\n\n candidates.append(base + \"/api/tags\") # Ollama format\n\n headers = _build_provider_auth_headers(api_key, request.provider_type)\n last_error = None\n\n try:\n async with httpx.AsyncClient(timeout=10.0) as client:\n for url in candidates:\n try:\n response = await client.get(url, headers=headers)\n\n if response.status_code != 200:\n if response.status_code != 404:\n last_error = f\"HTTP {response.status_code} from {url}: {response.text[:200]}\"\n continue\n\n body = response.json()\n raw = body.get(\"data\") if isinstance(body, dict) else body\n\n # Try OpenAI format: [{\"id\": \"model-name\", ...}, ...] or list of strings\n if isinstance(raw, list):\n models = sorted(\n m.get(\"id\", str(m)) if isinstance(m, dict) else str(m)\n for m in raw\n )\n if models:\n log.reason(\n f\"Found {len(models)} models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n # Try Ollama format: {\"models\": [{\"name\": \"model-name\", ...}, ...]}\n if isinstance(body, dict):\n models_list = body.get(\"models\", [])\n if isinstance(models_list, list):\n models = sorted(\n m.get(\"name\", str(m)) if isinstance(m, dict) else str(m)\n for m in models_list\n )\n if models:\n log.reason(\n f\"Found {len(models)} Ollama models at {url}\"\n )\n return FetchModelsResponse(\n models=models, total=len(models)\n )\n\n except httpx.TimeoutException:\n last_error = f\"Timeout at {url}\"\n except httpx.ConnectError:\n last_error = f\"Cannot connect at {url}\"\n except Exception as e:\n last_error = f\"Error at {url}: {str(e)[:100]}\"\n\n # All endpoints failed — return empty list with no error (user can type manually)\n log.explore(f\"No model endpoint responded for {request.base_url}. Last error: {last_error}\", error=f\"No model endpoint responded for {request.base_url}\")\n return FetchModelsResponse(models=[], total=0)\n\n except Exception as e:\n log.explore(f\"Unexpected error for {request.base_url}: {e}\", error=str(e))\n return FetchModelsResponse(models=[], total=0)\n\n\n# [/DEF:fetch_models_for_provider:Function]\n" }, { "contract_id": "test_connection", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 393, - "end_line": 444, + "start_line": 388, + "end_line": 437, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -21002,14 +21002,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_connection:Function]\n# @PURPOSE: Test connection to an LLM provider.\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMClient]\n@router.post(\"/providers/{provider_id}/test\")\nasync def test_connection(\n provider_id: str,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n logger.info(\n f\"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}\"\n )\n \"\"\"\n Test connection to an LLM provider.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n service = LLMProviderService(db)\n db_provider = service.get_provider(provider_id)\n if not db_provider:\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n\n api_key = service.get_decrypted_api_key(provider_id)\n\n # Check if API key was successfully decrypted\n if not api_key:\n logger.error(\n f\"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}\"\n )\n raise HTTPException(\n status_code=500,\n detail=\"Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.\",\n )\n\n client = LLMClient(\n provider_type=LLMProviderType(db_provider.provider_type),\n api_key=api_key,\n base_url=db_provider.base_url,\n default_model=db_provider.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_connection:Function]\n" + "body": "# [DEF:test_connection:Function]\n# @PURPOSE: Test connection to an LLM provider.\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: CALLS -> [LLMProviderService]\n# @RELATION: DEPENDS_ON -> [LLMClient]\n@router.post(\"/providers/{provider_id}/test\")\nasync def test_connection(\n provider_id: str,\n current_user: User = Depends(get_current_active_user),\n db: Session = Depends(get_db),\n):\n log.reason(\n f\"Testing connection for provider_id: {provider_id}\"\n )\n \"\"\"\n Test connection to an LLM provider.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n service = LLMProviderService(db)\n db_provider = service.get_provider(provider_id)\n if not db_provider:\n raise HTTPException(status_code=404, detail=\"Provider not found\")\n\n api_key = service.get_decrypted_api_key(provider_id)\n\n # Check if API key was successfully decrypted\n if not api_key:\n log.explore(f\"Failed to decrypt API key for provider {provider_id}\", error=\"API key decryption failed\")\n raise HTTPException(\n status_code=500,\n detail=\"Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.\",\n )\n\n client = LLMClient(\n provider_type=LLMProviderType(db_provider.provider_type),\n api_key=api_key,\n base_url=db_provider.base_url,\n default_model=db_provider.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_connection:Function]\n" }, { "contract_id": "test_provider_config", "contract_type": "Function", "file_path": "backend/src/api/routes/llm.py", - "start_line": 447, - "end_line": 486, + "start_line": 440, + "end_line": 479, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -21061,7 +21061,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_provider_config:Function]\n# @PURPOSE: Test connection with a provided configuration (not yet saved).\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: DEPENDS_ON -> [LLMClient]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.post(\"/providers/test\")\nasync def test_provider_config(\n config: LLMProviderConfig, current_user: User = Depends(get_current_active_user)\n):\n \"\"\"\n Test connection with a provided configuration.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n logger.info(\n f\"[llm_routes][test_provider_config][Action] Testing config for {config.name}\"\n )\n\n # Check if API key is provided\n if not config.api_key or config.api_key == \"********\":\n raise HTTPException(\n status_code=400, detail=\"API key is required for testing connection\"\n )\n\n client = LLMClient(\n provider_type=config.provider_type,\n api_key=config.api_key,\n base_url=config.base_url,\n default_model=config.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_provider_config:Function]\n" + "body": "# [DEF:test_provider_config:Function]\n# @PURPOSE: Test connection with a provided configuration (not yet saved).\n# @PRE: User is authenticated.\n# @POST: Returns success status and message.\n# @RELATION: DEPENDS_ON -> [LLMClient]\n# @RELATION: DEPENDS_ON -> [LLMProviderConfig]\n@router.post(\"/providers/test\")\nasync def test_provider_config(\n config: LLMProviderConfig, current_user: User = Depends(get_current_active_user)\n):\n \"\"\"\n Test connection with a provided configuration.\n \"\"\"\n from ...plugins.llm_analysis.service import LLMClient\n\n log.reason(\n f\"Testing config for {config.name}\"\n )\n\n # Check if API key is provided\n if not config.api_key or config.api_key == \"********\":\n raise HTTPException(\n status_code=400, detail=\"API key is required for testing connection\"\n )\n\n client = LLMClient(\n provider_type=config.provider_type,\n api_key=config.api_key,\n base_url=config.base_url,\n default_model=config.default_model,\n )\n\n try:\n await client.test_runtime_connection()\n return {\"success\": True, \"message\": \"Connection successful\"}\n except Exception as e:\n return {\"success\": False, \"error\": str(e)}\n\n\n# [/DEF:test_provider_config:Function]\n" }, { "contract_id": "MappingsApi", @@ -21289,7 +21289,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/migration.py", "start_line": 1, - "end_line": 400, + "end_line": 399, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -21414,7 +21414,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:MigrationApi:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: api, migration, dashboards, sync, dry-run\n# @PURPOSE: HTTP contract layer for migration orchestration, settings, dry-run, and mapping sync endpoints.\n# @LAYER: Infra\n# @RELATION: DEPENDS_ON ->[AppDependencies]\n# @RELATION: DEPENDS_ON ->[DatabaseModule]\n# @RELATION: DEPENDS_ON ->[DashboardSelection]\n# @RELATION: DEPENDS_ON ->[DashboardMetadata]\n# @RELATION: DEPENDS_ON ->[MigrationDryRunService]\n# @RELATION: DEPENDS_ON ->[IdMappingService]\n# @RELATION: DEPENDS_ON ->[ResourceMapping]\n# @INVARIANT: Migration endpoints never execute with invalid environment references and always return explicit HTTP errors on guard failures.\n# @PRE: Backend core services initialized and Database session available.\n# @POST: Migration tasks are enqueued or dry-run results are computed and returned.\n# @SIDE_EFFECT: Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls.\n# @DATA_CONTRACT: [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary]\n# @TEST_CONTRACT: [DashboardSelection + configured envs] -> [task_id | dry-run result | sync summary]\n# @TEST_SCENARIO: [invalid_environment] -> [HTTP_400_or_404]\n# @TEST_SCENARIO: [valid_execution] -> [success_payload_with_required_fields]\n# @TEST_EDGE: [missing_field] ->[HTTP_400]\n# @TEST_EDGE: [invalid_type] ->[validation_error]\n# @TEST_EDGE: [external_fail] ->[HTTP_500]\n# @TEST_INVARIANT: [EnvironmentValidationBeforeAction] -> VERIFIED_BY: [invalid_environment, valid_execution]\n\nfrom fastapi import APIRouter, Depends, HTTPException, Query\nfrom typing import List, Dict, Any, Optional, cast\nfrom sqlalchemy.orm import Session\nfrom ...dependencies import get_config_manager, get_task_manager, has_permission\nfrom ...core.database import get_db\nfrom ...models.dashboard import DashboardMetadata, DashboardSelection\nfrom ...core.superset_client import SupersetClient\nfrom ...core.logger import logger, belief_scope\nfrom ...core.migration.dry_run_orchestrator import MigrationDryRunService\nfrom ...core.mapping_service import IdMappingService\nfrom ...models.mapping import ResourceMapping\n\nlogger = cast(Any, logger)\n\nrouter = APIRouter(prefix=\"/api\", tags=[\"migration\"])\n\n\n# [DEF:get_dashboards:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch dashboard metadata from a requested environment for migration selection UI.\n# @PRE: env_id is provided and exists in configured environments.\n# @POST: Returns List[DashboardMetadata] for the resolved environment; emits HTTP_404 when environment is absent.\n# @SIDE_EFFECT: Reads environment configuration and performs remote Superset metadata retrieval over network.\n# @DATA_CONTRACT: Input[str env_id] -> Output[List[DashboardMetadata]]\n# @RELATION: CALLS ->[SupersetClient.get_dashboards_summary]\n@router.get(\"/environments/{env_id}/dashboards\", response_model=List[DashboardMetadata])\nasync def get_dashboards(\n env_id: str,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"get_dashboards\", f\"env_id={env_id}\"):\n logger.reason(f\"Fetching dashboards for environment: {env_id}\")\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n\n if not env:\n logger.explore(f\"Environment {env_id} not found in configuration\")\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary()\n logger.reflect(f\"Retrieved {len(dashboards)} dashboards from {env_id}\")\n return dashboards\n\n\n# [/DEF:get_dashboards:Function]\n\n\n# [DEF:execute_migration:Function]\n# @COMPLEXITY: 5\n# @PURPOSE: Validate migration selection and enqueue asynchronous migration task execution.\n# @PRE: DashboardSelection payload is valid and both source/target environments exist.\n# @POST: Returns {\"task_id\": str, \"message\": str} when task creation succeeds; emits HTTP_400/HTTP_500 on failure.\n# @SIDE_EFFECT: Reads configuration, writes task record through task manager, and writes operational logs.\n# @DATA_CONTRACT: Input[DashboardSelection] -> Output[Dict[str, str]]\n# @RELATION: CALLS ->[create_task]\n# @RELATION: DEPENDS_ON ->[DashboardSelection]\n# @INVARIANT: Migration task dispatch never occurs before source and target environment ids pass guard validation.\n@router.post(\"/migration/execute\")\nasync def execute_migration(\n selection: DashboardSelection,\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"execute_migration\"):\n logger.reason(\n f\"Initiating migration from {selection.source_env_id} to {selection.target_env_id}\"\n )\n\n # Validate environments exist\n environments = config_manager.get_environments()\n env_ids = {e.id for e in environments}\n\n if (\n selection.source_env_id not in env_ids\n or selection.target_env_id not in env_ids\n ):\n logger.explore(\n \"Invalid environment selection\",\n extra={\n \"source\": selection.source_env_id,\n \"target\": selection.target_env_id,\n },\n )\n raise HTTPException(\n status_code=400, detail=\"Invalid source or target environment\"\n )\n\n # Include replace_db_config and fix_cross_filters in the task parameters\n task_params = selection.dict()\n task_params[\"replace_db_config\"] = selection.replace_db_config\n task_params[\"fix_cross_filters\"] = selection.fix_cross_filters\n\n logger.reason(\n f\"Creating migration task with {len(selection.selected_ids)} dashboards\"\n )\n\n try:\n task = await task_manager.create_task(\"superset-migration\", task_params)\n logger.reflect(f\"Migration task created: {task.id}\")\n return {\"task_id\": task.id, \"message\": \"Migration initiated\"}\n except Exception as e:\n logger.explore(f\"Task creation failed: {e}\")\n raise HTTPException(\n status_code=500, detail=f\"Failed to create migration task: {str(e)}\"\n )\n\n\n# [/DEF:execute_migration:Function]\n\n\n# [DEF:dry_run_migration:Function]\n# @COMPLEXITY: 5\n# @PURPOSE: Build pre-flight migration diff and risk summary without mutating target systems.\n# @PRE: DashboardSelection is valid, source and target environments exist, differ, and selected_ids is non-empty.\n# @POST: Returns deterministic dry-run payload; emits HTTP_400 for guard violations and HTTP_500 for orchestrator value errors.\n# @SIDE_EFFECT: Reads local mappings from DB and fetches source/target metadata via Superset API.\n# @DATA_CONTRACT: Input[DashboardSelection] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[DashboardSelection]\n# @RELATION: DEPENDS_ON ->[MigrationDryRunService]\n# @INVARIANT: Dry-run flow remains read-only and rejects identical source/target environments before service execution.\n@router.post(\"/migration/dry-run\", response_model=Dict[str, Any])\nasync def dry_run_migration(\n selection: DashboardSelection,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"dry_run_migration\"):\n logger.reason(\n f\"Starting dry run: {selection.source_env_id} -> {selection.target_env_id}\"\n )\n\n environments = config_manager.get_environments()\n env_map = {env.id: env for env in environments}\n source_env = env_map.get(selection.source_env_id)\n target_env = env_map.get(selection.target_env_id)\n\n if not source_env or not target_env:\n logger.explore(\"Invalid environment selection for dry run\")\n raise HTTPException(\n status_code=400, detail=\"Invalid source or target environment\"\n )\n\n if selection.source_env_id == selection.target_env_id:\n logger.explore(\"Source and target environments are identical\")\n raise HTTPException(\n status_code=400,\n detail=\"Source and target environments must be different\",\n )\n\n if not selection.selected_ids:\n logger.explore(\"No dashboards selected for dry run\")\n raise HTTPException(\n status_code=400, detail=\"No dashboards selected for dry run\"\n )\n\n service = MigrationDryRunService()\n source_client = SupersetClient(source_env)\n target_client = SupersetClient(target_env)\n\n try:\n result = service.run(\n selection=selection,\n source_client=source_client,\n target_client=target_client,\n db=db,\n )\n logger.reflect(\"Dry run analysis complete\")\n return result\n except ValueError as exc:\n logger.explore(f\"Dry run orchestrator failed: {exc}\")\n raise HTTPException(status_code=500, detail=str(exc)) from exc\n\n\n# [/DEF:dry_run_migration:Function]\n\n\n# [DEF:get_migration_settings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Read and return configured migration synchronization cron expression.\n# @PRE: Configuration store is available and requester has READ permission.\n# @POST: Returns {\"cron\": str} reflecting current persisted settings value.\n# @SIDE_EFFECT: Reads configuration from config manager.\n# @DATA_CONTRACT: Input[None] -> Output[Dict[str, str]]\n# @RELATION: DEPENDS_ON ->[AppDependencies]\n@router.get(\"/migration/settings\", response_model=Dict[str, str])\nasync def get_migration_settings(\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\"get_migration_settings\"):\n config = config_manager.get_config()\n cron = config.settings.migration_sync_cron\n return {\"cron\": cron}\n\n\n# [/DEF:get_migration_settings:Function]\n\n\n# [DEF:update_migration_settings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Validate and persist migration synchronization cron expression update.\n# @PRE: Payload includes \"cron\" key and requester has WRITE permission.\n# @POST: Returns {\"cron\": str, \"status\": \"updated\"} and persists updated cron value.\n# @SIDE_EFFECT: Mutates configuration and writes persisted config through config manager.\n# @DATA_CONTRACT: Input[Dict[str, str]] -> Output[Dict[str, str]]\n# @RELATION: DEPENDS_ON ->[AppDependencies]\n@router.put(\"/migration/settings\", response_model=Dict[str, str])\nasync def update_migration_settings(\n payload: Dict[str, str],\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:migration\", \"WRITE\")),\n):\n with belief_scope(\"update_migration_settings\"):\n if \"cron\" not in payload:\n raise HTTPException(\n status_code=400, detail=\"Missing 'cron' field in payload\"\n )\n\n cron_expr = payload[\"cron\"]\n\n config = config_manager.get_config()\n config.settings.migration_sync_cron = cron_expr\n config_manager.save_config(config)\n\n return {\"cron\": cron_expr, \"status\": \"updated\"}\n\n\n# [/DEF:update_migration_settings:Function]\n\n\n# [DEF:get_resource_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch synchronized resource mappings with optional filters and pagination for migration mappings view.\n# @PRE: skip>=0, 1<=limit<=500, DB session is active, requester has READ permission.\n# @POST: Returns {\"items\": [...], \"total\": int} where items reflect applied filters and pagination.\n# @SIDE_EFFECT: Executes database read queries against ResourceMapping table.\n# @DATA_CONTRACT: Input[QueryParams] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[ResourceMapping]\n@router.get(\"/migration/mappings-data\", response_model=Dict[str, Any])\nasync def get_resource_mappings(\n skip: int = Query(0, ge=0),\n limit: int = Query(50, ge=1, le=500),\n search: Optional[str] = Query(None, description=\"Search by resource name or UUID\"),\n env_id: Optional[str] = Query(None, description=\"Filter by environment ID\"),\n resource_type: Optional[str] = Query(None, description=\"Filter by resource type\"),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\"get_resource_mappings\"):\n query = db.query(ResourceMapping)\n\n if env_id:\n query = query.filter(ResourceMapping.environment_id == env_id)\n\n if resource_type:\n query = query.filter(ResourceMapping.resource_type == resource_type.upper())\n\n if search:\n search_term = f\"%{search}%\"\n query = query.filter(\n (ResourceMapping.resource_name.ilike(search_term))\n | (ResourceMapping.uuid.ilike(search_term))\n )\n\n total = query.count()\n mappings = (\n query.order_by(ResourceMapping.resource_type, ResourceMapping.resource_name)\n .offset(skip)\n .limit(limit)\n .all()\n )\n\n items = []\n for m in mappings:\n mapping = cast(Any, m)\n resource_type_value = (\n mapping.resource_type.value\n if mapping.resource_type is not None\n else None\n )\n last_synced_at = (\n mapping.last_synced_at.isoformat()\n if mapping.last_synced_at is not None\n else None\n )\n items.append(\n {\n \"id\": mapping.id,\n \"environment_id\": mapping.environment_id,\n \"resource_type\": resource_type_value,\n \"uuid\": mapping.uuid,\n \"remote_id\": mapping.remote_integer_id,\n \"resource_name\": mapping.resource_name,\n \"last_synced_at\": last_synced_at,\n }\n )\n\n return {\"items\": items, \"total\": total}\n\n\n# [/DEF:get_resource_mappings:Function]\n\n\n# [DEF:trigger_sync_now:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Trigger immediate ID synchronization for every configured environment.\n# @PRE: At least one environment is configured and requester has EXECUTE permission.\n# @POST: Returns sync summary with synced/failed counts after attempting all environments.\n# @SIDE_EFFECT: Upserts Environment rows, commits DB transaction, performs network sync calls, and writes logs.\n# @DATA_CONTRACT: Input[None] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[IdMappingService]\n# @RELATION: CALLS ->[sync_environment]\n@router.post(\"/migration/sync-now\", response_model=Dict[str, Any])\nasync def trigger_sync_now(\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"trigger_sync_now\"):\n from ...core.logger import logger\n from ...models.mapping import Environment as EnvironmentModel\n\n config = config_manager.get_config()\n environments = config.environments\n\n if not environments:\n raise HTTPException(status_code=400, detail=\"No environments configured\")\n\n # Ensure each environment exists in DB (upsert) to satisfy FK constraints\n for env in environments:\n existing = db.query(EnvironmentModel).filter_by(id=env.id).first()\n if not existing:\n db_env = EnvironmentModel(\n id=env.id,\n name=env.name,\n url=env.url,\n credentials_id=env.id, # Use env.id as credentials reference\n )\n db.add(db_env)\n logger.info(\n f\"[trigger_sync_now][Action] Created environment row for {env.id}\"\n )\n else:\n existing.name = env.name\n existing.url = env.url\n db.commit()\n\n service = IdMappingService(db)\n results = {\"synced\": [], \"failed\": []}\n\n for env in environments:\n try:\n client = SupersetClient(env)\n service.sync_environment(env.id, client)\n results[\"synced\"].append(env.id)\n logger.info(f\"[trigger_sync_now][Action] Synced environment {env.id}\")\n except Exception as e:\n results[\"failed\"].append({\"env_id\": env.id, \"error\": str(e)})\n logger.error(f\"[trigger_sync_now][Error] Failed to sync {env.id}: {e}\")\n\n return {\n \"status\": \"completed\",\n \"synced_count\": len(results[\"synced\"]),\n \"failed_count\": len(results[\"failed\"]),\n \"details\": results,\n }\n\n\n# [/DEF:trigger_sync_now:Function]\n\n# [/DEF:MigrationApi:Module]\n" + "body": "# [DEF:MigrationApi:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: api, migration, dashboards, sync, dry-run\n# @PURPOSE: HTTP contract layer for migration orchestration, settings, dry-run, and mapping sync endpoints.\n# @LAYER: Infra\n# @RELATION: DEPENDS_ON ->[AppDependencies]\n# @RELATION: DEPENDS_ON ->[DatabaseModule]\n# @RELATION: DEPENDS_ON ->[DashboardSelection]\n# @RELATION: DEPENDS_ON ->[DashboardMetadata]\n# @RELATION: DEPENDS_ON ->[MigrationDryRunService]\n# @RELATION: DEPENDS_ON ->[IdMappingService]\n# @RELATION: DEPENDS_ON ->[ResourceMapping]\n# @INVARIANT: Migration endpoints never execute with invalid environment references and always return explicit HTTP errors on guard failures.\n# @PRE: Backend core services initialized and Database session available.\n# @POST: Migration tasks are enqueued or dry-run results are computed and returned.\n# @SIDE_EFFECT: Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls.\n# @DATA_CONTRACT: [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary]\n# @TEST_CONTRACT: [DashboardSelection + configured envs] -> [task_id | dry-run result | sync summary]\n# @TEST_SCENARIO: [invalid_environment] -> [HTTP_400_or_404]\n# @TEST_SCENARIO: [valid_execution] -> [success_payload_with_required_fields]\n# @TEST_EDGE: [missing_field] ->[HTTP_400]\n# @TEST_EDGE: [invalid_type] ->[validation_error]\n# @TEST_EDGE: [external_fail] ->[HTTP_500]\n# @TEST_INVARIANT: [EnvironmentValidationBeforeAction] -> VERIFIED_BY: [invalid_environment, valid_execution]\n\nfrom fastapi import APIRouter, Depends, HTTPException, Query\nfrom typing import List, Dict, Any, Optional, cast\nfrom sqlalchemy.orm import Session\nfrom ...dependencies import get_config_manager, get_task_manager, has_permission\nfrom ...core.database import get_db\nfrom ...models.dashboard import DashboardMetadata, DashboardSelection\nfrom ...core.superset_client import SupersetClient\nfrom ...core.logger import logger, belief_scope\nfrom ...core.migration.dry_run_orchestrator import MigrationDryRunService\nfrom ...core.mapping_service import IdMappingService\nfrom ...models.mapping import ResourceMapping\n\nlogger = cast(Any, logger)\n\nrouter = APIRouter(prefix=\"/api\", tags=[\"migration\"])\n\n\n# [DEF:get_dashboards:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch dashboard metadata from a requested environment for migration selection UI.\n# @PRE: env_id is provided and exists in configured environments.\n# @POST: Returns List[DashboardMetadata] for the resolved environment; emits HTTP_404 when environment is absent.\n# @SIDE_EFFECT: Reads environment configuration and performs remote Superset metadata retrieval over network.\n# @DATA_CONTRACT: Input[str env_id] -> Output[List[DashboardMetadata]]\n# @RELATION: CALLS ->[SupersetClient.get_dashboards_summary]\n@router.get(\"/environments/{env_id}/dashboards\", response_model=List[DashboardMetadata])\nasync def get_dashboards(\n env_id: str,\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"get_dashboards\", f\"env_id={env_id}\"):\n logger.reason(f\"Fetching dashboards for environment: {env_id}\")\n environments = config_manager.get_environments()\n env = next((e for e in environments if e.id == env_id), None)\n\n if not env:\n logger.explore(f\"Environment {env_id} not found in configuration\")\n raise HTTPException(status_code=404, detail=\"Environment not found\")\n\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary()\n logger.reflect(f\"Retrieved {len(dashboards)} dashboards from {env_id}\")\n return dashboards\n\n\n# [/DEF:get_dashboards:Function]\n\n\n# [DEF:execute_migration:Function]\n# @COMPLEXITY: 5\n# @PURPOSE: Validate migration selection and enqueue asynchronous migration task execution.\n# @PRE: DashboardSelection payload is valid and both source/target environments exist.\n# @POST: Returns {\"task_id\": str, \"message\": str} when task creation succeeds; emits HTTP_400/HTTP_500 on failure.\n# @SIDE_EFFECT: Reads configuration, writes task record through task manager, and writes operational logs.\n# @DATA_CONTRACT: Input[DashboardSelection] -> Output[Dict[str, str]]\n# @RELATION: CALLS ->[create_task]\n# @RELATION: DEPENDS_ON ->[DashboardSelection]\n# @INVARIANT: Migration task dispatch never occurs before source and target environment ids pass guard validation.\n@router.post(\"/migration/execute\")\nasync def execute_migration(\n selection: DashboardSelection,\n config_manager=Depends(get_config_manager),\n task_manager=Depends(get_task_manager),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"execute_migration\"):\n logger.reason(\n f\"Initiating migration from {selection.source_env_id} to {selection.target_env_id}\"\n )\n\n # Validate environments exist\n environments = config_manager.get_environments()\n env_ids = {e.id for e in environments}\n\n if (\n selection.source_env_id not in env_ids\n or selection.target_env_id not in env_ids\n ):\n logger.explore(\n \"Invalid environment selection\",\n extra={\n \"source\": selection.source_env_id,\n \"target\": selection.target_env_id,\n },\n )\n raise HTTPException(\n status_code=400, detail=\"Invalid source or target environment\"\n )\n\n # Include replace_db_config and fix_cross_filters in the task parameters\n task_params = selection.dict()\n task_params[\"replace_db_config\"] = selection.replace_db_config\n task_params[\"fix_cross_filters\"] = selection.fix_cross_filters\n\n logger.reason(\n f\"Creating migration task with {len(selection.selected_ids)} dashboards\"\n )\n\n try:\n task = await task_manager.create_task(\"superset-migration\", task_params)\n logger.reflect(f\"Migration task created: {task.id}\")\n return {\"task_id\": task.id, \"message\": \"Migration initiated\"}\n except Exception as e:\n logger.explore(f\"Task creation failed: {e}\")\n raise HTTPException(\n status_code=500, detail=f\"Failed to create migration task: {str(e)}\"\n )\n\n\n# [/DEF:execute_migration:Function]\n\n\n# [DEF:dry_run_migration:Function]\n# @COMPLEXITY: 5\n# @PURPOSE: Build pre-flight migration diff and risk summary without mutating target systems.\n# @PRE: DashboardSelection is valid, source and target environments exist, differ, and selected_ids is non-empty.\n# @POST: Returns deterministic dry-run payload; emits HTTP_400 for guard violations and HTTP_500 for orchestrator value errors.\n# @SIDE_EFFECT: Reads local mappings from DB and fetches source/target metadata via Superset API.\n# @DATA_CONTRACT: Input[DashboardSelection] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[DashboardSelection]\n# @RELATION: DEPENDS_ON ->[MigrationDryRunService]\n# @INVARIANT: Dry-run flow remains read-only and rejects identical source/target environments before service execution.\n@router.post(\"/migration/dry-run\", response_model=Dict[str, Any])\nasync def dry_run_migration(\n selection: DashboardSelection,\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"dry_run_migration\"):\n logger.reason(\n f\"Starting dry run: {selection.source_env_id} -> {selection.target_env_id}\"\n )\n\n environments = config_manager.get_environments()\n env_map = {env.id: env for env in environments}\n source_env = env_map.get(selection.source_env_id)\n target_env = env_map.get(selection.target_env_id)\n\n if not source_env or not target_env:\n logger.explore(\"Invalid environment selection for dry run\")\n raise HTTPException(\n status_code=400, detail=\"Invalid source or target environment\"\n )\n\n if selection.source_env_id == selection.target_env_id:\n logger.explore(\"Source and target environments are identical\")\n raise HTTPException(\n status_code=400,\n detail=\"Source and target environments must be different\",\n )\n\n if not selection.selected_ids:\n logger.explore(\"No dashboards selected for dry run\")\n raise HTTPException(\n status_code=400, detail=\"No dashboards selected for dry run\"\n )\n\n service = MigrationDryRunService()\n source_client = SupersetClient(source_env)\n target_client = SupersetClient(target_env)\n\n try:\n result = service.run(\n selection=selection,\n source_client=source_client,\n target_client=target_client,\n db=db,\n )\n logger.reflect(\"Dry run analysis complete\")\n return result\n except ValueError as exc:\n logger.explore(f\"Dry run orchestrator failed: {exc}\")\n raise HTTPException(status_code=500, detail=str(exc)) from exc\n\n\n# [/DEF:dry_run_migration:Function]\n\n\n# [DEF:get_migration_settings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Read and return configured migration synchronization cron expression.\n# @PRE: Configuration store is available and requester has READ permission.\n# @POST: Returns {\"cron\": str} reflecting current persisted settings value.\n# @SIDE_EFFECT: Reads configuration from config manager.\n# @DATA_CONTRACT: Input[None] -> Output[Dict[str, str]]\n# @RELATION: DEPENDS_ON ->[AppDependencies]\n@router.get(\"/migration/settings\", response_model=Dict[str, str])\nasync def get_migration_settings(\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\"get_migration_settings\"):\n config = config_manager.get_config()\n cron = config.settings.migration_sync_cron\n return {\"cron\": cron}\n\n\n# [/DEF:get_migration_settings:Function]\n\n\n# [DEF:update_migration_settings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Validate and persist migration synchronization cron expression update.\n# @PRE: Payload includes \"cron\" key and requester has WRITE permission.\n# @POST: Returns {\"cron\": str, \"status\": \"updated\"} and persists updated cron value.\n# @SIDE_EFFECT: Mutates configuration and writes persisted config through config manager.\n# @DATA_CONTRACT: Input[Dict[str, str]] -> Output[Dict[str, str]]\n# @RELATION: DEPENDS_ON ->[AppDependencies]\n@router.put(\"/migration/settings\", response_model=Dict[str, str])\nasync def update_migration_settings(\n payload: Dict[str, str],\n config_manager=Depends(get_config_manager),\n _=Depends(has_permission(\"plugin:migration\", \"WRITE\")),\n):\n with belief_scope(\"update_migration_settings\"):\n if \"cron\" not in payload:\n raise HTTPException(\n status_code=400, detail=\"Missing 'cron' field in payload\"\n )\n\n cron_expr = payload[\"cron\"]\n\n config = config_manager.get_config()\n config.settings.migration_sync_cron = cron_expr\n config_manager.save_config(config)\n\n return {\"cron\": cron_expr, \"status\": \"updated\"}\n\n\n# [/DEF:update_migration_settings:Function]\n\n\n# [DEF:get_resource_mappings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Fetch synchronized resource mappings with optional filters and pagination for migration mappings view.\n# @PRE: skip>=0, 1<=limit<=500, DB session is active, requester has READ permission.\n# @POST: Returns {\"items\": [...], \"total\": int} where items reflect applied filters and pagination.\n# @SIDE_EFFECT: Executes database read queries against ResourceMapping table.\n# @DATA_CONTRACT: Input[QueryParams] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[ResourceMapping]\n@router.get(\"/migration/mappings-data\", response_model=Dict[str, Any])\nasync def get_resource_mappings(\n skip: int = Query(0, ge=0),\n limit: int = Query(50, ge=1, le=500),\n search: Optional[str] = Query(None, description=\"Search by resource name or UUID\"),\n env_id: Optional[str] = Query(None, description=\"Filter by environment ID\"),\n resource_type: Optional[str] = Query(None, description=\"Filter by resource type\"),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"READ\")),\n):\n with belief_scope(\"get_resource_mappings\"):\n query = db.query(ResourceMapping)\n\n if env_id:\n query = query.filter(ResourceMapping.environment_id == env_id)\n\n if resource_type:\n query = query.filter(ResourceMapping.resource_type == resource_type.upper())\n\n if search:\n search_term = f\"%{search}%\"\n query = query.filter(\n (ResourceMapping.resource_name.ilike(search_term))\n | (ResourceMapping.uuid.ilike(search_term))\n )\n\n total = query.count()\n mappings = (\n query.order_by(ResourceMapping.resource_type, ResourceMapping.resource_name)\n .offset(skip)\n .limit(limit)\n .all()\n )\n\n items = []\n for m in mappings:\n mapping = cast(Any, m)\n resource_type_value = (\n mapping.resource_type.value\n if mapping.resource_type is not None\n else None\n )\n last_synced_at = (\n mapping.last_synced_at.isoformat()\n if mapping.last_synced_at is not None\n else None\n )\n items.append(\n {\n \"id\": mapping.id,\n \"environment_id\": mapping.environment_id,\n \"resource_type\": resource_type_value,\n \"uuid\": mapping.uuid,\n \"remote_id\": mapping.remote_integer_id,\n \"resource_name\": mapping.resource_name,\n \"last_synced_at\": last_synced_at,\n }\n )\n\n return {\"items\": items, \"total\": total}\n\n\n# [/DEF:get_resource_mappings:Function]\n\n\n# [DEF:trigger_sync_now:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Trigger immediate ID synchronization for every configured environment.\n# @PRE: At least one environment is configured and requester has EXECUTE permission.\n# @POST: Returns sync summary with synced/failed counts after attempting all environments.\n# @SIDE_EFFECT: Upserts Environment rows, commits DB transaction, performs network sync calls, and writes logs.\n# @DATA_CONTRACT: Input[None] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[IdMappingService]\n# @RELATION: CALLS ->[sync_environment]\n@router.post(\"/migration/sync-now\", response_model=Dict[str, Any])\nasync def trigger_sync_now(\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"trigger_sync_now\"):\n from ...models.mapping import Environment as EnvironmentModel\n\n config = config_manager.get_config()\n environments = config.environments\n\n if not environments:\n raise HTTPException(status_code=400, detail=\"No environments configured\")\n\n # Ensure each environment exists in DB (upsert) to satisfy FK constraints\n for env in environments:\n existing = db.query(EnvironmentModel).filter_by(id=env.id).first()\n if not existing:\n db_env = EnvironmentModel(\n id=env.id,\n name=env.name,\n url=env.url,\n credentials_id=env.id, # Use env.id as credentials reference\n )\n db.add(db_env)\n logger.reason(\n f\"Created environment row for {env.id}\"\n )\n else:\n existing.name = env.name\n existing.url = env.url\n db.commit()\n\n service = IdMappingService(db)\n results = {\"synced\": [], \"failed\": []}\n\n for env in environments:\n try:\n client = SupersetClient(env)\n service.sync_environment(env.id, client)\n results[\"synced\"].append(env.id)\n logger.reason(f\"Synced environment {env.id}\")\n except Exception as e:\n results[\"failed\"].append({\"env_id\": env.id, \"error\": str(e)})\n logger.explore(f\"Failed to sync {env.id}\", error=str(e))\n\n return {\n \"status\": \"completed\",\n \"synced_count\": len(results[\"synced\"]),\n \"failed_count\": len(results[\"failed\"]),\n \"details\": results,\n }\n\n\n# [/DEF:trigger_sync_now:Function]\n\n# [/DEF:MigrationApi:Module]\n" }, { "contract_id": "execute_migration", @@ -21686,7 +21686,7 @@ "contract_type": "Function", "file_path": "backend/src/api/routes/migration.py", "start_line": 333, - "end_line": 398, + "end_line": 397, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -21750,7 +21750,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:trigger_sync_now:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Trigger immediate ID synchronization for every configured environment.\n# @PRE: At least one environment is configured and requester has EXECUTE permission.\n# @POST: Returns sync summary with synced/failed counts after attempting all environments.\n# @SIDE_EFFECT: Upserts Environment rows, commits DB transaction, performs network sync calls, and writes logs.\n# @DATA_CONTRACT: Input[None] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[IdMappingService]\n# @RELATION: CALLS ->[sync_environment]\n@router.post(\"/migration/sync-now\", response_model=Dict[str, Any])\nasync def trigger_sync_now(\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"trigger_sync_now\"):\n from ...core.logger import logger\n from ...models.mapping import Environment as EnvironmentModel\n\n config = config_manager.get_config()\n environments = config.environments\n\n if not environments:\n raise HTTPException(status_code=400, detail=\"No environments configured\")\n\n # Ensure each environment exists in DB (upsert) to satisfy FK constraints\n for env in environments:\n existing = db.query(EnvironmentModel).filter_by(id=env.id).first()\n if not existing:\n db_env = EnvironmentModel(\n id=env.id,\n name=env.name,\n url=env.url,\n credentials_id=env.id, # Use env.id as credentials reference\n )\n db.add(db_env)\n logger.info(\n f\"[trigger_sync_now][Action] Created environment row for {env.id}\"\n )\n else:\n existing.name = env.name\n existing.url = env.url\n db.commit()\n\n service = IdMappingService(db)\n results = {\"synced\": [], \"failed\": []}\n\n for env in environments:\n try:\n client = SupersetClient(env)\n service.sync_environment(env.id, client)\n results[\"synced\"].append(env.id)\n logger.info(f\"[trigger_sync_now][Action] Synced environment {env.id}\")\n except Exception as e:\n results[\"failed\"].append({\"env_id\": env.id, \"error\": str(e)})\n logger.error(f\"[trigger_sync_now][Error] Failed to sync {env.id}: {e}\")\n\n return {\n \"status\": \"completed\",\n \"synced_count\": len(results[\"synced\"]),\n \"failed_count\": len(results[\"failed\"]),\n \"details\": results,\n }\n\n\n# [/DEF:trigger_sync_now:Function]\n" + "body": "# [DEF:trigger_sync_now:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Trigger immediate ID synchronization for every configured environment.\n# @PRE: At least one environment is configured and requester has EXECUTE permission.\n# @POST: Returns sync summary with synced/failed counts after attempting all environments.\n# @SIDE_EFFECT: Upserts Environment rows, commits DB transaction, performs network sync calls, and writes logs.\n# @DATA_CONTRACT: Input[None] -> Output[Dict[str, Any]]\n# @RELATION: DEPENDS_ON ->[IdMappingService]\n# @RELATION: CALLS ->[sync_environment]\n@router.post(\"/migration/sync-now\", response_model=Dict[str, Any])\nasync def trigger_sync_now(\n config_manager=Depends(get_config_manager),\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"plugin:migration\", \"EXECUTE\")),\n):\n with belief_scope(\"trigger_sync_now\"):\n from ...models.mapping import Environment as EnvironmentModel\n\n config = config_manager.get_config()\n environments = config.environments\n\n if not environments:\n raise HTTPException(status_code=400, detail=\"No environments configured\")\n\n # Ensure each environment exists in DB (upsert) to satisfy FK constraints\n for env in environments:\n existing = db.query(EnvironmentModel).filter_by(id=env.id).first()\n if not existing:\n db_env = EnvironmentModel(\n id=env.id,\n name=env.name,\n url=env.url,\n credentials_id=env.id, # Use env.id as credentials reference\n )\n db.add(db_env)\n logger.reason(\n f\"Created environment row for {env.id}\"\n )\n else:\n existing.name = env.name\n existing.url = env.url\n db.commit()\n\n service = IdMappingService(db)\n results = {\"synced\": [], \"failed\": []}\n\n for env in environments:\n try:\n client = SupersetClient(env)\n service.sync_environment(env.id, client)\n results[\"synced\"].append(env.id)\n logger.reason(f\"Synced environment {env.id}\")\n except Exception as e:\n results[\"failed\"].append({\"env_id\": env.id, \"error\": str(e)})\n logger.explore(f\"Failed to sync {env.id}\", error=str(e))\n\n return {\n \"status\": \"completed\",\n \"synced_count\": len(results[\"synced\"]),\n \"failed_count\": len(results[\"failed\"]),\n \"details\": results,\n }\n\n\n# [/DEF:trigger_sync_now:Function]\n" }, { "contract_id": "PluginsRouter", @@ -21881,7 +21881,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/profile.py", "start_line": 1, - "end_line": 151, + "end_line": 154, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -22015,14 +22015,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:ProfileApiModule:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: api, profile, preferences, self-service, account-lookup\n# @PURPOSE: Exposes self-scoped profile preference endpoints and environment-based Superset account lookup.\n# @LAYER: API\n# @RELATION: [DEPENDS_ON] ->[ProfileService]\n# @RELATION: [DEPENDS_ON] ->[get_current_user]\n# @RELATION: [DEPENDS_ON] ->[get_db]\n#\n# @INVARIANT: Endpoints are self-scoped and never mutate another user preference.\n# @UX_STATE: ProfileLoad -> Returns stable ProfilePreferenceResponse for authenticated user.\n# @UX_STATE: Saving -> Validation errors map to actionable 422 details.\n# @UX_STATE: LookupLoading -> Returns success/degraded Superset lookup payload.\n# @UX_FEEDBACK: Stable status/message/warning payloads support profile page feedback.\n# @UX_RECOVERY: Lookup degradation keeps manual username save path available.\n\n# [SECTION: IMPORTS]\nfrom typing import Optional\n\nfrom fastapi import APIRouter, Depends, HTTPException, Query\nfrom sqlalchemy.orm import Session\n\nfrom ...core.database import get_db\nfrom ...core.logger import logger, belief_scope\nfrom ...dependencies import (\n get_config_manager,\n get_current_user,\n get_plugin_loader,\n)\nfrom ...models.auth import User\nfrom ...schemas.profile import (\n ProfilePreferenceResponse,\n ProfilePreferenceUpdateRequest,\n SupersetAccountLookupRequest,\n SupersetAccountLookupResponse,\n)\nfrom ...services.profile_service import (\n EnvironmentNotFoundError,\n ProfileAuthorizationError,\n ProfileService,\n ProfileValidationError,\n)\n# [/SECTION]\n\nrouter = APIRouter(prefix=\"/api/profile\", tags=[\"profile\"])\n\n\n# [DEF:_get_profile_service:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Build profile service for current request scope.\n# @PRE: db session and config manager are available.\n# @POST: Returns a ready ProfileService instance.\ndef _get_profile_service(db: Session, config_manager, plugin_loader=None) -> ProfileService:\n return ProfileService(\n db=db,\n config_manager=config_manager,\n plugin_loader=plugin_loader,\n )\n# [/DEF:_get_profile_service:Function]\n\n\n# [DEF:get_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Get authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and authenticated user context.\n# @POST: Returns preference payload for current user only.\n@router.get(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def get_preferences(\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.get_preferences\", f\"user_id={current_user.id}\"):\n logger.reason(\"[REASON] Resolving current user preference\")\n service = _get_profile_service(db, config_manager, plugin_loader)\n return service.get_my_preference(current_user)\n# [/DEF:get_preferences:Function]\n\n\n# [DEF:update_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Update authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and valid request payload.\n# @POST: Persists normalized preference for current user or raises validation/authorization errors.\n@router.patch(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def update_preferences(\n payload: ProfilePreferenceUpdateRequest,\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.update_preferences\", f\"user_id={current_user.id}\"):\n service = _get_profile_service(db, config_manager, plugin_loader)\n try:\n logger.reason(\"[REASON] Attempting preference save\")\n return service.update_my_preference(current_user=current_user, payload=payload)\n except ProfileValidationError as exc:\n logger.reflect(\"[REFLECT] Preference validation failed\")\n raise HTTPException(status_code=422, detail=exc.errors) from exc\n except ProfileAuthorizationError as exc:\n logger.explore(\"[EXPLORE] Cross-user mutation guard blocked request\")\n raise HTTPException(status_code=403, detail=str(exc)) from exc\n# [/DEF:update_preferences:Function]\n\n\n# [DEF:lookup_superset_accounts:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Lookup Superset account candidates in selected environment.\n# @PRE: Valid JWT, authenticated context, and environment_id query parameter.\n# @POST: Returns success or degraded lookup payload with stable shape.\n@router.get(\"/superset-accounts\", response_model=SupersetAccountLookupResponse)\nasync def lookup_superset_accounts(\n environment_id: str = Query(...),\n search: Optional[str] = Query(default=None),\n page_index: int = Query(default=0, ge=0),\n page_size: int = Query(default=20, ge=1, le=100),\n sort_column: str = Query(default=\"username\"),\n sort_order: str = Query(default=\"desc\"),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\n \"profile.lookup_superset_accounts\",\n f\"user_id={current_user.id}, environment_id={environment_id}\",\n ):\n service = _get_profile_service(db, config_manager, plugin_loader)\n lookup_request = SupersetAccountLookupRequest(\n environment_id=environment_id,\n search=search,\n page_index=page_index,\n page_size=page_size,\n sort_column=sort_column,\n sort_order=sort_order,\n )\n try:\n logger.reason(\"[REASON] Executing Superset account lookup\")\n return service.lookup_superset_accounts(\n current_user=current_user,\n request=lookup_request,\n )\n except EnvironmentNotFoundError as exc:\n logger.explore(\"[EXPLORE] Lookup request references unknown environment\")\n raise HTTPException(status_code=404, detail=str(exc)) from exc\n# [/DEF:lookup_superset_accounts:Function]\n\n# [/DEF:ProfileApiModule:Module]\n" + "body": "# [DEF:ProfileApiModule:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: api, profile, preferences, self-service, account-lookup\n# @PURPOSE: Exposes self-scoped profile preference endpoints and environment-based Superset account lookup.\n# @LAYER: API\n# @RELATION: [DEPENDS_ON] ->[ProfileService]\n# @RELATION: [DEPENDS_ON] ->[get_current_user]\n# @RELATION: [DEPENDS_ON] ->[get_db]\n#\n# @INVARIANT: Endpoints are self-scoped and never mutate another user preference.\n# @UX_STATE: ProfileLoad -> Returns stable ProfilePreferenceResponse for authenticated user.\n# @UX_STATE: Saving -> Validation errors map to actionable 422 details.\n# @UX_STATE: LookupLoading -> Returns success/degraded Superset lookup payload.\n# @UX_FEEDBACK: Stable status/message/warning payloads support profile page feedback.\n# @UX_RECOVERY: Lookup degradation keeps manual username save path available.\n\n# [SECTION: IMPORTS]\nfrom typing import Optional\n\nfrom fastapi import APIRouter, Depends, HTTPException, Query\nfrom sqlalchemy.orm import Session\n\nfrom ...core.database import get_db\nfrom ...core.logger import belief_scope\nfrom ...core.cot_logger import MarkerLogger\nfrom ...dependencies import (\n get_config_manager,\n get_current_user,\n get_plugin_loader,\n)\nfrom ...models.auth import User\nfrom ...schemas.profile import (\n ProfilePreferenceResponse,\n ProfilePreferenceUpdateRequest,\n SupersetAccountLookupRequest,\n SupersetAccountLookupResponse,\n)\nfrom ...services.profile_service import (\n EnvironmentNotFoundError,\n ProfileAuthorizationError,\n ProfileService,\n ProfileValidationError,\n)\n# [/SECTION]\n\nlog = MarkerLogger(\"Profile\")\n\nrouter = APIRouter(prefix=\"/api/profile\", tags=[\"profile\"])\n\n\n# [DEF:_get_profile_service:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Build profile service for current request scope.\n# @PRE: db session and config manager are available.\n# @POST: Returns a ready ProfileService instance.\ndef _get_profile_service(db: Session, config_manager, plugin_loader=None) -> ProfileService:\n return ProfileService(\n db=db,\n config_manager=config_manager,\n plugin_loader=plugin_loader,\n )\n# [/DEF:_get_profile_service:Function]\n\n\n# [DEF:get_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Get authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and authenticated user context.\n# @POST: Returns preference payload for current user only.\n@router.get(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def get_preferences(\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.get_preferences\", f\"user_id={current_user.id}\"):\n log.reason(\"Resolving current user preference\")\n service = _get_profile_service(db, config_manager, plugin_loader)\n return service.get_my_preference(current_user)\n# [/DEF:get_preferences:Function]\n\n\n# [DEF:update_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Update authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and valid request payload.\n# @POST: Persists normalized preference for current user or raises validation/authorization errors.\n@router.patch(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def update_preferences(\n payload: ProfilePreferenceUpdateRequest,\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.update_preferences\", f\"user_id={current_user.id}\"):\n service = _get_profile_service(db, config_manager, plugin_loader)\n try:\n log.reason(\"Attempting preference save\")\n return service.update_my_preference(current_user=current_user, payload=payload)\n except ProfileValidationError as exc:\n log.reflect(\"Preference validation failed\")\n raise HTTPException(status_code=422, detail=exc.errors) from exc\n except ProfileAuthorizationError as exc:\n log.explore(\"Cross-user mutation guard blocked request\", error=\"User attempted to mutate another user's resource\")\n raise HTTPException(status_code=403, detail=str(exc)) from exc\n# [/DEF:update_preferences:Function]\n\n\n# [DEF:lookup_superset_accounts:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Lookup Superset account candidates in selected environment.\n# @PRE: Valid JWT, authenticated context, and environment_id query parameter.\n# @POST: Returns success or degraded lookup payload with stable shape.\n@router.get(\"/superset-accounts\", response_model=SupersetAccountLookupResponse)\nasync def lookup_superset_accounts(\n environment_id: str = Query(...),\n search: Optional[str] = Query(default=None),\n page_index: int = Query(default=0, ge=0),\n page_size: int = Query(default=20, ge=1, le=100),\n sort_column: str = Query(default=\"username\"),\n sort_order: str = Query(default=\"desc\"),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\n \"profile.lookup_superset_accounts\",\n f\"user_id={current_user.id}, environment_id={environment_id}\",\n ):\n service = _get_profile_service(db, config_manager, plugin_loader)\n lookup_request = SupersetAccountLookupRequest(\n environment_id=environment_id,\n search=search,\n page_index=page_index,\n page_size=page_size,\n sort_column=sort_column,\n sort_order=sort_order,\n )\n try:\n log.reason(\"Executing Superset account lookup\")\n return service.lookup_superset_accounts(\n current_user=current_user,\n request=lookup_request,\n )\n except EnvironmentNotFoundError as exc:\n log.explore(\"Lookup request references unknown environment\", error=\"EnvironmentNotFoundError raised during Superset account lookup\")\n raise HTTPException(status_code=404, detail=str(exc)) from exc\n# [/DEF:lookup_superset_accounts:Function]\n\n# [/DEF:ProfileApiModule:Module]\n" }, { "contract_id": "_get_profile_service", "contract_type": "Function", "file_path": "backend/src/api/routes/profile.py", - "start_line": 49, - "end_line": 60, + "start_line": 52, + "end_line": 63, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -22074,8 +22074,8 @@ "contract_id": "get_preferences", "contract_type": "Function", "file_path": "backend/src/api/routes/profile.py", - "start_line": 63, - "end_line": 79, + "start_line": 66, + "end_line": 82, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -22121,14 +22121,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Get authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and authenticated user context.\n# @POST: Returns preference payload for current user only.\n@router.get(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def get_preferences(\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.get_preferences\", f\"user_id={current_user.id}\"):\n logger.reason(\"[REASON] Resolving current user preference\")\n service = _get_profile_service(db, config_manager, plugin_loader)\n return service.get_my_preference(current_user)\n# [/DEF:get_preferences:Function]\n" + "body": "# [DEF:get_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Get authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and authenticated user context.\n# @POST: Returns preference payload for current user only.\n@router.get(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def get_preferences(\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.get_preferences\", f\"user_id={current_user.id}\"):\n log.reason(\"Resolving current user preference\")\n service = _get_profile_service(db, config_manager, plugin_loader)\n return service.get_my_preference(current_user)\n# [/DEF:get_preferences:Function]\n" }, { "contract_id": "update_preferences", "contract_type": "Function", "file_path": "backend/src/api/routes/profile.py", - "start_line": 82, - "end_line": 106, + "start_line": 85, + "end_line": 109, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -22174,14 +22174,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:update_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Update authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and valid request payload.\n# @POST: Persists normalized preference for current user or raises validation/authorization errors.\n@router.patch(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def update_preferences(\n payload: ProfilePreferenceUpdateRequest,\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.update_preferences\", f\"user_id={current_user.id}\"):\n service = _get_profile_service(db, config_manager, plugin_loader)\n try:\n logger.reason(\"[REASON] Attempting preference save\")\n return service.update_my_preference(current_user=current_user, payload=payload)\n except ProfileValidationError as exc:\n logger.reflect(\"[REFLECT] Preference validation failed\")\n raise HTTPException(status_code=422, detail=exc.errors) from exc\n except ProfileAuthorizationError as exc:\n logger.explore(\"[EXPLORE] Cross-user mutation guard blocked request\")\n raise HTTPException(status_code=403, detail=str(exc)) from exc\n# [/DEF:update_preferences:Function]\n" + "body": "# [DEF:update_preferences:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Update authenticated user's dashboard filter preference.\n# @PRE: Valid JWT and valid request payload.\n# @POST: Persists normalized preference for current user or raises validation/authorization errors.\n@router.patch(\"/preferences\", response_model=ProfilePreferenceResponse)\nasync def update_preferences(\n payload: ProfilePreferenceUpdateRequest,\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\"profile.update_preferences\", f\"user_id={current_user.id}\"):\n service = _get_profile_service(db, config_manager, plugin_loader)\n try:\n log.reason(\"Attempting preference save\")\n return service.update_my_preference(current_user=current_user, payload=payload)\n except ProfileValidationError as exc:\n log.reflect(\"Preference validation failed\")\n raise HTTPException(status_code=422, detail=exc.errors) from exc\n except ProfileAuthorizationError as exc:\n log.explore(\"Cross-user mutation guard blocked request\", error=\"User attempted to mutate another user's resource\")\n raise HTTPException(status_code=403, detail=str(exc)) from exc\n# [/DEF:update_preferences:Function]\n" }, { "contract_id": "lookup_superset_accounts", "contract_type": "Function", "file_path": "backend/src/api/routes/profile.py", - "start_line": 109, - "end_line": 149, + "start_line": 112, + "end_line": 152, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -22227,7 +22227,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:lookup_superset_accounts:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Lookup Superset account candidates in selected environment.\n# @PRE: Valid JWT, authenticated context, and environment_id query parameter.\n# @POST: Returns success or degraded lookup payload with stable shape.\n@router.get(\"/superset-accounts\", response_model=SupersetAccountLookupResponse)\nasync def lookup_superset_accounts(\n environment_id: str = Query(...),\n search: Optional[str] = Query(default=None),\n page_index: int = Query(default=0, ge=0),\n page_size: int = Query(default=20, ge=1, le=100),\n sort_column: str = Query(default=\"username\"),\n sort_order: str = Query(default=\"desc\"),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\n \"profile.lookup_superset_accounts\",\n f\"user_id={current_user.id}, environment_id={environment_id}\",\n ):\n service = _get_profile_service(db, config_manager, plugin_loader)\n lookup_request = SupersetAccountLookupRequest(\n environment_id=environment_id,\n search=search,\n page_index=page_index,\n page_size=page_size,\n sort_column=sort_column,\n sort_order=sort_order,\n )\n try:\n logger.reason(\"[REASON] Executing Superset account lookup\")\n return service.lookup_superset_accounts(\n current_user=current_user,\n request=lookup_request,\n )\n except EnvironmentNotFoundError as exc:\n logger.explore(\"[EXPLORE] Lookup request references unknown environment\")\n raise HTTPException(status_code=404, detail=str(exc)) from exc\n# [/DEF:lookup_superset_accounts:Function]\n" + "body": "# [DEF:lookup_superset_accounts:Function]\n# @RELATION: CALLS -> ProfileService\n# @PURPOSE: Lookup Superset account candidates in selected environment.\n# @PRE: Valid JWT, authenticated context, and environment_id query parameter.\n# @POST: Returns success or degraded lookup payload with stable shape.\n@router.get(\"/superset-accounts\", response_model=SupersetAccountLookupResponse)\nasync def lookup_superset_accounts(\n environment_id: str = Query(...),\n search: Optional[str] = Query(default=None),\n page_index: int = Query(default=0, ge=0),\n page_size: int = Query(default=20, ge=1, le=100),\n sort_column: str = Query(default=\"username\"),\n sort_order: str = Query(default=\"desc\"),\n current_user: User = Depends(get_current_user),\n db: Session = Depends(get_db),\n config_manager=Depends(get_config_manager),\n plugin_loader=Depends(get_plugin_loader),\n):\n with belief_scope(\n \"profile.lookup_superset_accounts\",\n f\"user_id={current_user.id}, environment_id={environment_id}\",\n ):\n service = _get_profile_service(db, config_manager, plugin_loader)\n lookup_request = SupersetAccountLookupRequest(\n environment_id=environment_id,\n search=search,\n page_index=page_index,\n page_size=page_size,\n sort_column=sort_column,\n sort_order=sort_order,\n )\n try:\n log.reason(\"Executing Superset account lookup\")\n return service.lookup_superset_accounts(\n current_user=current_user,\n request=lookup_request,\n )\n except EnvironmentNotFoundError as exc:\n log.explore(\"Lookup request references unknown environment\", error=\"EnvironmentNotFoundError raised during Superset account lookup\")\n raise HTTPException(status_code=404, detail=str(exc)) from exc\n# [/DEF:lookup_superset_accounts:Function]\n" }, { "contract_id": "ReportsRouter", @@ -22613,7 +22613,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/settings.py", "start_line": 1, - "end_line": 653, + "end_line": 644, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -22732,14 +22732,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SettingsRouter:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: settings, api, router, fastapi\n# @PURPOSE: Provides API endpoints for managing application settings and Superset environments.\n# @LAYER: API\n# @RELATION: [DEPENDS_ON] ->[ConfigManager]\n# @RELATION: [DEPENDS_ON] ->[get_config_manager]\n# @RELATION: [DEPENDS_ON] ->[has_permission]\n#\n# @INVARIANT: All settings changes must be persisted via ConfigManager.\n# @PUBLIC_API: router\n\n# [SECTION: IMPORTS]\nfrom fastapi import APIRouter, Depends, HTTPException\nfrom typing import List\nfrom pydantic import BaseModel\nfrom ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig\nfrom ...models.storage import StorageConfig\nfrom ...dependencies import get_config_manager, has_permission\nfrom ...core.config_manager import ConfigManager\nfrom ...core.logger import logger, belief_scope\nfrom ...core.superset_client import SupersetClient\nfrom ...services.llm_prompt_templates import normalize_llm_settings\nfrom ...models.llm import ValidationPolicy\nfrom ...models.config import AppConfigRecord\nfrom ...schemas.settings import (\n ValidationPolicyCreate,\n ValidationPolicyUpdate,\n ValidationPolicyResponse,\n)\nfrom ...core.database import get_db\nfrom sqlalchemy.orm import Session\n# [/SECTION]\n\n\n# [DEF:LoggingConfigResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response model for logging configuration with current task log level.\n# @SEMANTICS: logging, config, response\nclass LoggingConfigResponse(BaseModel):\n level: str\n task_log_level: str\n enable_belief_state: bool\n\n\n# [/DEF:LoggingConfigResponse:Class]\n\nrouter = APIRouter()\n\n\n# [DEF:_normalize_superset_env_url:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Canonicalize Superset environment URL to base host/path without trailing /api/v1.\n# @PRE: raw_url can be empty.\n# @POST: Returns normalized base URL.\ndef _normalize_superset_env_url(raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[: -len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n\n\n# [/DEF:_normalize_superset_env_url:Function]\n\n\n# [DEF:_validate_superset_connection_fast:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Run lightweight Superset connectivity validation without full pagination scan.\n# @PRE: env contains valid URL and credentials.\n# @POST: Raises on auth/API failures; returns None on success.\ndef _validate_superset_connection_fast(env: Environment) -> None:\n client = SupersetClient(env)\n # 1) Explicit auth check\n client.authenticate()\n # 2) Single lightweight API call to ensure read access\n client.get_dashboards_page(\n query={\n \"page\": 0,\n \"page_size\": 1,\n \"columns\": [\"id\"],\n }\n )\n\n\n# [/DEF:_validate_superset_connection_fast:Function]\n\n\n# [DEF:get_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves all application settings.\n# @PRE: Config manager is available.\n# @POST: Returns masked AppConfig.\n# @RETURN: AppConfig - The current configuration.\n@router.get(\"\", response_model=AppConfig)\nasync def get_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_settings\"):\n logger.info(\"[get_settings][Entry] Fetching all settings\")\n config = config_manager.get_config().copy(deep=True)\n config.settings.llm = normalize_llm_settings(config.settings.llm)\n # Mask passwords\n for env in config.environments:\n if env.password:\n env.password = \"********\"\n return config\n\n\n# [/DEF:get_settings:Function]\n\n\n# [DEF:get_features:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Public endpoint returning feature flags for frontend sidebar filtering.\n# @RATIONALE: Unauthenticated because sidebar filtering must work for all users, not just admins.\n# @PRE: Config manager is available.\n# @POST: Returns dict with dataset_review and health_monitor booleans.\n@router.get(\"/features\")\nasync def get_features(\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n return config_manager.get_config().settings.features.model_dump()\n\n\n# [/DEF:get_features:Function]\n\n\n# [DEF:update_global_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates global application settings.\n# @PRE: New settings are provided.\n# @POST: Global settings are updated.\n# @PARAM: settings (GlobalSettings) - The new global settings.\n# @RETURN: GlobalSettings - The updated settings.\n@router.patch(\"/global\", response_model=GlobalSettings)\nasync def update_global_settings(\n settings: GlobalSettings,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_global_settings\"):\n logger.info(\"[update_global_settings][Entry] Updating global settings\")\n\n config_manager.update_global_settings(settings)\n return settings\n\n\n# [/DEF:update_global_settings:Function]\n\n\n# [DEF:get_storage_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves storage-specific settings.\n# @RETURN: StorageConfig - The storage configuration.\n@router.get(\"/storage\", response_model=StorageConfig)\nasync def get_storage_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_storage_settings\"):\n return config_manager.get_config().settings.storage\n\n\n# [/DEF:get_storage_settings:Function]\n\n\n# [DEF:update_storage_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates storage-specific settings.\n# @PARAM: storage (StorageConfig) - The new storage settings.\n# @POST: Storage settings are updated and saved.\n# @RETURN: StorageConfig - The updated storage settings.\n@router.put(\"/storage\", response_model=StorageConfig)\nasync def update_storage_settings(\n storage: StorageConfig,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_storage_settings\"):\n is_valid, message = config_manager.validate_path(storage.root_path)\n if not is_valid:\n raise HTTPException(status_code=400, detail=message)\n\n settings = config_manager.get_config().settings\n settings.storage = storage\n config_manager.update_global_settings(settings)\n return config_manager.get_config().settings.storage\n\n\n# [/DEF:update_storage_settings:Function]\n\n\n# [DEF:get_environments:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Lists all configured Superset environments.\n# @PRE: Config manager is available.\n# @POST: Returns list of environments.\n# @RETURN: List[Environment] - List of environments.\n@router.get(\"/environments\", response_model=List[Environment])\nasync def get_environments(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_environments\"):\n logger.info(\"[get_environments][Entry] Fetching environments\")\n environments = config_manager.get_environments()\n return [\n env.copy(update={\"url\": _normalize_superset_env_url(env.url)})\n for env in environments\n ]\n\n\n# [/DEF:get_environments:Function]\n\n\n# [DEF:add_environment:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Adds a new Superset environment.\n# @PRE: Environment data is valid and reachable.\n# @POST: Environment is added to config.\n# @PARAM: env (Environment) - The environment to add.\n# @RETURN: Environment - The added environment.\n@router.post(\"/environments\", response_model=Environment)\nasync def add_environment(\n env: Environment,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"add_environment\"):\n logger.info(f\"[add_environment][Entry] Adding environment {env.id}\")\n env = env.copy(update={\"url\": _normalize_superset_env_url(env.url)})\n\n # Validate connection before adding (fast path)\n try:\n _validate_superset_connection_fast(env)\n except Exception as e:\n logger.error(\n f\"[add_environment][Coherence:Failed] Connection validation failed: {e}\"\n )\n raise HTTPException(\n status_code=400, detail=f\"Connection validation failed: {e}\"\n )\n\n config_manager.add_environment(env)\n return env\n\n\n# [/DEF:add_environment:Function]\n\n\n# [DEF:update_environment:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates an existing Superset environment.\n# @PRE: ID and valid environment data are provided.\n# @POST: Environment is updated in config.\n# @PARAM: id (str) - The ID of the environment to update.\n# @PARAM: env (Environment) - The updated environment data.\n# @RETURN: Environment - The updated environment.\n@router.put(\"/environments/{id}\", response_model=Environment)\nasync def update_environment(\n id: str,\n env: Environment,\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n with belief_scope(\"update_environment\"):\n logger.info(f\"[update_environment][Entry] Updating environment {id}\")\n\n env = env.copy(update={\"url\": _normalize_superset_env_url(env.url)})\n\n # If password is masked, we need the real one for validation\n env_to_validate = env.copy(deep=True)\n if env_to_validate.password == \"********\":\n old_env = next(\n (e for e in config_manager.get_environments() if e.id == id), None\n )\n if old_env:\n env_to_validate.password = old_env.password\n\n # Validate connection before updating (fast path)\n try:\n _validate_superset_connection_fast(env_to_validate)\n except Exception as e:\n logger.error(\n f\"[update_environment][Coherence:Failed] Connection validation failed: {e}\"\n )\n raise HTTPException(\n status_code=400, detail=f\"Connection validation failed: {e}\"\n )\n\n if config_manager.update_environment(id, env):\n return env\n raise HTTPException(status_code=404, detail=f\"Environment {id} not found\")\n\n\n# [/DEF:update_environment:Function]\n\n\n# [DEF:delete_environment:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Deletes a Superset environment.\n# @PRE: ID is provided.\n# @POST: Environment is removed from config.\n# @PARAM: id (str) - The ID of the environment to delete.\n@router.delete(\"/environments/{id}\")\nasync def delete_environment(\n id: str, config_manager: ConfigManager = Depends(get_config_manager)\n):\n with belief_scope(\"delete_environment\"):\n logger.info(f\"[delete_environment][Entry] Deleting environment {id}\")\n config_manager.delete_environment(id)\n return {\"message\": f\"Environment {id} deleted\"}\n\n\n# [/DEF:delete_environment:Function]\n\n\n# [DEF:test_environment_connection:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Tests the connection to a Superset environment.\n# @PRE: ID is provided.\n# @POST: Returns success or error status.\n# @PARAM: id (str) - The ID of the environment to test.\n# @RETURN: dict - Success message or error.\n@router.post(\"/environments/{id}/test\")\nasync def test_environment_connection(\n id: str, config_manager: ConfigManager = Depends(get_config_manager)\n):\n with belief_scope(\"test_environment_connection\"):\n logger.info(f\"[test_environment_connection][Entry] Testing environment {id}\")\n\n # Find environment\n env = next((e for e in config_manager.get_environments() if e.id == id), None)\n if not env:\n raise HTTPException(status_code=404, detail=f\"Environment {id} not found\")\n\n try:\n _validate_superset_connection_fast(env)\n\n logger.info(\n f\"[test_environment_connection][Coherence:OK] Connection successful for {id}\"\n )\n return {\"status\": \"success\", \"message\": \"Connection successful\"}\n except Exception as e:\n logger.error(\n f\"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}\"\n )\n return {\"status\": \"error\", \"message\": str(e)}\n\n\n# [/DEF:test_environment_connection:Function]\n\n\n# [DEF:get_logging_config:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves current logging configuration.\n# @PRE: Config manager is available.\n# @POST: Returns logging configuration.\n# @RETURN: LoggingConfigResponse - The current logging config.\n@router.get(\"/logging\", response_model=LoggingConfigResponse)\nasync def get_logging_config(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_logging_config\"):\n logging_config = config_manager.get_config().settings.logging\n return LoggingConfigResponse(\n level=logging_config.level,\n task_log_level=logging_config.task_log_level,\n enable_belief_state=logging_config.enable_belief_state,\n )\n\n\n# [/DEF:get_logging_config:Function]\n\n\n# [DEF:update_logging_config:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates logging configuration.\n# @PRE: New logging config is provided.\n# @POST: Logging configuration is updated and saved.\n# @PARAM: config (LoggingConfig) - The new logging configuration.\n# @RETURN: LoggingConfigResponse - The updated logging config.\n@router.patch(\"/logging\", response_model=LoggingConfigResponse)\nasync def update_logging_config(\n config: LoggingConfig,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_logging_config\"):\n logger.info(\n f\"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}\"\n )\n\n # Get current settings and update logging config\n settings = config_manager.get_config().settings\n settings.logging = config\n config_manager.update_global_settings(settings)\n\n return LoggingConfigResponse(\n level=config.level,\n task_log_level=config.task_log_level,\n enable_belief_state=config.enable_belief_state,\n )\n\n\n# [/DEF:update_logging_config:Function]\n\n\n# [DEF:ConsolidatedSettingsResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response model for consolidated application settings.\nclass ConsolidatedSettingsResponse(BaseModel):\n environments: List[dict]\n connections: List[dict]\n llm: dict\n llm_providers: List[dict]\n logging: dict\n storage: dict\n notifications: dict = {}\n features: dict = {}\n\n\n# [/DEF:ConsolidatedSettingsResponse:Class]\n\n\n# [DEF:get_consolidated_settings:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Retrieves all settings categories in a single call.\n# @PRE: Config manager is available and the caller holds admin settings read permission.\n# @POST: Returns consolidated settings, provider metadata, and persisted notification payload in one stable response.\n# @SIDE_EFFECT: Opens one database session to read LLM providers and config-backed notification payload, then closes it.\n# @DATA_CONTRACT: Input[ConfigManager] -> Output[ConsolidatedSettingsResponse]\n# @RELATION: [DEPENDS_ON] ->[ConfigManager]\n# @RELATION: [DEPENDS_ON] ->[LLMProviderService]\n# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]\n# @RELATION: [DEPENDS_ON] ->[SessionLocal]\n# @RELATION: [DEPENDS_ON] ->[has_permission]\n# @RELATION: [DEPENDS_ON] ->[normalize_llm_settings]\n@router.get(\"/consolidated\", response_model=ConsolidatedSettingsResponse)\nasync def get_consolidated_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_consolidated_settings\"):\n logger.reason(\"Fetching consolidated settings payload\")\n\n config = config_manager.get_config()\n\n from ...services.llm_provider import LLMProviderService\n from ...core.database import SessionLocal\n\n db = SessionLocal()\n notifications_payload = {}\n try:\n llm_service = LLMProviderService(db)\n providers = llm_service.get_all_providers()\n llm_providers_list = [\n {\n \"id\": p.id,\n \"provider_type\": p.provider_type,\n \"name\": p.name,\n \"base_url\": p.base_url,\n \"api_key\": \"********\",\n \"default_model\": p.default_model,\n \"is_active\": p.is_active,\n }\n for p in providers\n ]\n\n config_record = (\n db.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n )\n if config_record and isinstance(config_record.payload, dict):\n notifications_payload = (\n config_record.payload.get(\"notifications\", {}) or {}\n )\n finally:\n db.close()\n\n normalized_llm = normalize_llm_settings(config.settings.llm)\n\n response_payload = ConsolidatedSettingsResponse(\n environments=[env.dict() for env in config.environments],\n connections=config.settings.connections,\n llm=normalized_llm,\n llm_providers=llm_providers_list,\n logging=config.settings.logging.dict(),\n storage=config.settings.storage.dict(),\n notifications=notifications_payload,\n features=config.settings.features.model_dump(),\n )\n logger.reflect(\n \"Consolidated settings payload assembled\",\n extra={\n \"environment_count\": len(response_payload.environments),\n \"provider_count\": len(response_payload.llm_providers),\n },\n )\n return response_payload\n\n\n# [/DEF:get_consolidated_settings:Function]\n\n\n# [DEF:update_consolidated_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Bulk update application settings from the consolidated view.\n# @PRE: User has admin permissions, config is valid.\n# @POST: Settings are updated and saved via ConfigManager.\n@router.patch(\"/consolidated\")\nasync def update_consolidated_settings(\n settings_patch: dict,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_consolidated_settings\"):\n logger.info(\n \"[update_consolidated_settings][Entry] Applying consolidated settings patch\"\n )\n\n current_config = config_manager.get_config()\n current_settings = current_config.settings\n\n # Update connections if provided\n if \"connections\" in settings_patch:\n current_settings.connections = settings_patch[\"connections\"]\n\n # Update LLM if provided\n if \"llm\" in settings_patch:\n current_settings.llm = normalize_llm_settings(settings_patch[\"llm\"])\n\n # Update Logging if provided\n if \"logging\" in settings_patch:\n current_settings.logging = LoggingConfig(**settings_patch[\"logging\"])\n\n # Update Storage if provided\n if \"storage\" in settings_patch:\n new_storage = StorageConfig(**settings_patch[\"storage\"])\n is_valid, message = config_manager.validate_path(new_storage.root_path)\n if not is_valid:\n raise HTTPException(status_code=400, detail=message)\n current_settings.storage = new_storage\n\n # Update Features if provided\n if \"features\" in settings_patch:\n from ...core.config_models import FeaturesConfig\n\n current_settings.features = FeaturesConfig(**settings_patch[\"features\"])\n\n if \"notifications\" in settings_patch:\n payload = config_manager.get_payload()\n payload[\"notifications\"] = settings_patch[\"notifications\"]\n config_manager.save_config(payload)\n\n config_manager.update_global_settings(current_settings)\n return {\"status\": \"success\", \"message\": \"Settings updated\"}\n\n\n# [/DEF:update_consolidated_settings:Function]\n\n\n# [DEF:get_validation_policies:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Lists all validation policies.\n# @RETURN: List[ValidationPolicyResponse] - List of policies.\n@router.get(\"/automation/policies\", response_model=List[ValidationPolicyResponse])\nasync def get_validation_policies(\n db: Session = Depends(get_db), _=Depends(has_permission(\"admin:settings\", \"READ\"))\n):\n with belief_scope(\"get_validation_policies\"):\n return db.query(ValidationPolicy).all()\n\n\n# [/DEF:get_validation_policies:Function]\n\n\n# [DEF:create_validation_policy:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Creates a new validation policy.\n# @PARAM: policy (ValidationPolicyCreate) - The policy data.\n# @RETURN: ValidationPolicyResponse - The created policy.\n@router.post(\"/automation/policies\", response_model=ValidationPolicyResponse)\nasync def create_validation_policy(\n policy: ValidationPolicyCreate,\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"create_validation_policy\"):\n db_policy = ValidationPolicy(**policy.dict())\n db.add(db_policy)\n db.commit()\n db.refresh(db_policy)\n return db_policy\n\n\n# [/DEF:create_validation_policy:Function]\n\n\n# [DEF:update_validation_policy:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates an existing validation policy.\n# @PARAM: id (str) - The ID of the policy to update.\n# @PARAM: policy (ValidationPolicyUpdate) - The updated policy data.\n# @RETURN: ValidationPolicyResponse - The updated policy.\n@router.patch(\"/automation/policies/{id}\", response_model=ValidationPolicyResponse)\nasync def update_validation_policy(\n id: str,\n policy: ValidationPolicyUpdate,\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_validation_policy\"):\n db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first()\n if not db_policy:\n raise HTTPException(status_code=404, detail=\"Policy not found\")\n\n update_data = policy.dict(exclude_unset=True)\n for key, value in update_data.items():\n setattr(db_policy, key, value)\n\n db.commit()\n db.refresh(db_policy)\n return db_policy\n\n\n# [/DEF:update_validation_policy:Function]\n\n\n# [DEF:delete_validation_policy:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Deletes a validation policy.\n# @PARAM: id (str) - The ID of the policy to delete.\n@router.delete(\"/automation/policies/{id}\")\nasync def delete_validation_policy(\n id: str,\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"delete_validation_policy\"):\n db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first()\n if not db_policy:\n raise HTTPException(status_code=404, detail=\"Policy not found\")\n\n db.delete(db_policy)\n db.commit()\n return {\"message\": \"Policy deleted\"}\n\n\n# [/DEF:delete_validation_policy:Function]\n\n# [/DEF:SettingsRouter:Module]\n" + "body": "# [DEF:SettingsRouter:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: settings, api, router, fastapi\n# @PURPOSE: Provides API endpoints for managing application settings and Superset environments.\n# @LAYER: API\n# @RELATION: [DEPENDS_ON] ->[ConfigManager]\n# @RELATION: [DEPENDS_ON] ->[get_config_manager]\n# @RELATION: [DEPENDS_ON] ->[has_permission]\n#\n# @INVARIANT: All settings changes must be persisted via ConfigManager.\n# @PUBLIC_API: router\n\n# [SECTION: IMPORTS]\nfrom fastapi import APIRouter, Depends, HTTPException\nfrom typing import List\nfrom pydantic import BaseModel\nfrom ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig\nfrom ...models.storage import StorageConfig\nfrom ...dependencies import get_config_manager, has_permission\nfrom ...core.config_manager import ConfigManager\nfrom ...core.logger import belief_scope\nfrom ...core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"Settings\")\nfrom ...core.superset_client import SupersetClient\nfrom ...services.llm_prompt_templates import normalize_llm_settings\nfrom ...models.llm import ValidationPolicy\nfrom ...models.config import AppConfigRecord\nfrom ...schemas.settings import (\n ValidationPolicyCreate,\n ValidationPolicyUpdate,\n ValidationPolicyResponse,\n)\nfrom ...core.database import get_db\nfrom sqlalchemy.orm import Session\n# [/SECTION]\n\n\n# [DEF:LoggingConfigResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response model for logging configuration with current task log level.\n# @SEMANTICS: logging, config, response\nclass LoggingConfigResponse(BaseModel):\n level: str\n task_log_level: str\n enable_belief_state: bool\n\n\n# [/DEF:LoggingConfigResponse:Class]\n\nrouter = APIRouter()\n\n\n# [DEF:_normalize_superset_env_url:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Canonicalize Superset environment URL to base host/path without trailing /api/v1.\n# @PRE: raw_url can be empty.\n# @POST: Returns normalized base URL.\ndef _normalize_superset_env_url(raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[: -len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n\n\n# [/DEF:_normalize_superset_env_url:Function]\n\n\n# [DEF:_validate_superset_connection_fast:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Run lightweight Superset connectivity validation without full pagination scan.\n# @PRE: env contains valid URL and credentials.\n# @POST: Raises on auth/API failures; returns None on success.\ndef _validate_superset_connection_fast(env: Environment) -> None:\n client = SupersetClient(env)\n # 1) Explicit auth check\n client.authenticate()\n # 2) Single lightweight API call to ensure read access\n client.get_dashboards_page(\n query={\n \"page\": 0,\n \"page_size\": 1,\n \"columns\": [\"id\"],\n }\n )\n\n\n# [/DEF:_validate_superset_connection_fast:Function]\n\n\n# [DEF:get_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves all application settings.\n# @PRE: Config manager is available.\n# @POST: Returns masked AppConfig.\n# @RETURN: AppConfig - The current configuration.\n@router.get(\"\", response_model=AppConfig)\nasync def get_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_settings\"):\n log.reason(\"Fetching all settings\")\n config = config_manager.get_config().copy(deep=True)\n config.settings.llm = normalize_llm_settings(config.settings.llm)\n # Mask passwords\n for env in config.environments:\n if env.password:\n env.password = \"********\"\n return config\n\n\n# [/DEF:get_settings:Function]\n\n\n# [DEF:get_features:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Public endpoint returning feature flags for frontend sidebar filtering.\n# @RATIONALE: Unauthenticated because sidebar filtering must work for all users, not just admins.\n# @PRE: Config manager is available.\n# @POST: Returns dict with dataset_review and health_monitor booleans.\n@router.get(\"/features\")\nasync def get_features(\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n return config_manager.get_config().settings.features.model_dump()\n\n\n# [/DEF:get_features:Function]\n\n\n# [DEF:update_global_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates global application settings.\n# @PRE: New settings are provided.\n# @POST: Global settings are updated.\n# @PARAM: settings (GlobalSettings) - The new global settings.\n# @RETURN: GlobalSettings - The updated settings.\n@router.patch(\"/global\", response_model=GlobalSettings)\nasync def update_global_settings(\n settings: GlobalSettings,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_global_settings\"):\n log.reason(\"Updating global settings\")\n\n config_manager.update_global_settings(settings)\n return settings\n\n\n# [/DEF:update_global_settings:Function]\n\n\n# [DEF:get_storage_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves storage-specific settings.\n# @RETURN: StorageConfig - The storage configuration.\n@router.get(\"/storage\", response_model=StorageConfig)\nasync def get_storage_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_storage_settings\"):\n return config_manager.get_config().settings.storage\n\n\n# [/DEF:get_storage_settings:Function]\n\n\n# [DEF:update_storage_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates storage-specific settings.\n# @PARAM: storage (StorageConfig) - The new storage settings.\n# @POST: Storage settings are updated and saved.\n# @RETURN: StorageConfig - The updated storage settings.\n@router.put(\"/storage\", response_model=StorageConfig)\nasync def update_storage_settings(\n storage: StorageConfig,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_storage_settings\"):\n is_valid, message = config_manager.validate_path(storage.root_path)\n if not is_valid:\n raise HTTPException(status_code=400, detail=message)\n\n settings = config_manager.get_config().settings\n settings.storage = storage\n config_manager.update_global_settings(settings)\n return config_manager.get_config().settings.storage\n\n\n# [/DEF:update_storage_settings:Function]\n\n\n# [DEF:get_environments:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Lists all configured Superset environments.\n# @PRE: Config manager is available.\n# @POST: Returns list of environments.\n# @RETURN: List[Environment] - List of environments.\n@router.get(\"/environments\", response_model=List[Environment])\nasync def get_environments(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_environments\"):\n log.reason(\"Fetching environments\")\n environments = config_manager.get_environments()\n return [\n env.copy(update={\"url\": _normalize_superset_env_url(env.url)})\n for env in environments\n ]\n\n\n# [/DEF:get_environments:Function]\n\n\n# [DEF:add_environment:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Adds a new Superset environment.\n# @PRE: Environment data is valid and reachable.\n# @POST: Environment is added to config.\n# @PARAM: env (Environment) - The environment to add.\n# @RETURN: Environment - The added environment.\n@router.post(\"/environments\", response_model=Environment)\nasync def add_environment(\n env: Environment,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"add_environment\"):\n log.reason(\"Adding environment\", payload={\"env_id\": env.id})\n env = env.copy(update={\"url\": _normalize_superset_env_url(env.url)})\n\n # Validate connection before adding (fast path)\n try:\n _validate_superset_connection_fast(env)\n except Exception as e:\n log.explore(\"Connection validation failed\", payload={\"env_id\": env.id}, error=str(e))\n raise HTTPException(\n status_code=400, detail=f\"Connection validation failed: {e}\"\n )\n\n config_manager.add_environment(env)\n return env\n\n\n# [/DEF:add_environment:Function]\n\n\n# [DEF:update_environment:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates an existing Superset environment.\n# @PRE: ID and valid environment data are provided.\n# @POST: Environment is updated in config.\n# @PARAM: id (str) - The ID of the environment to update.\n# @PARAM: env (Environment) - The updated environment data.\n# @RETURN: Environment - The updated environment.\n@router.put(\"/environments/{id}\", response_model=Environment)\nasync def update_environment(\n id: str,\n env: Environment,\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n with belief_scope(\"update_environment\"):\n log.reason(\"Updating environment\", payload={\"id\": id})\n\n env = env.copy(update={\"url\": _normalize_superset_env_url(env.url)})\n\n # If password is masked, we need the real one for validation\n env_to_validate = env.copy(deep=True)\n if env_to_validate.password == \"********\":\n old_env = next(\n (e for e in config_manager.get_environments() if e.id == id), None\n )\n if old_env:\n env_to_validate.password = old_env.password\n\n # Validate connection before updating (fast path)\n try:\n _validate_superset_connection_fast(env_to_validate)\n except Exception as e:\n log.explore(\"Connection validation failed\", payload={\"id\": id}, error=str(e))\n raise HTTPException(\n status_code=400, detail=f\"Connection validation failed: {e}\"\n )\n\n if config_manager.update_environment(id, env):\n return env\n raise HTTPException(status_code=404, detail=f\"Environment {id} not found\")\n\n\n# [/DEF:update_environment:Function]\n\n\n# [DEF:delete_environment:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Deletes a Superset environment.\n# @PRE: ID is provided.\n# @POST: Environment is removed from config.\n# @PARAM: id (str) - The ID of the environment to delete.\n@router.delete(\"/environments/{id}\")\nasync def delete_environment(\n id: str, config_manager: ConfigManager = Depends(get_config_manager)\n):\n with belief_scope(\"delete_environment\"):\n log.reason(\"Deleting environment\", payload={\"id\": id})\n config_manager.delete_environment(id)\n return {\"message\": f\"Environment {id} deleted\"}\n\n\n# [/DEF:delete_environment:Function]\n\n\n# [DEF:test_environment_connection:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Tests the connection to a Superset environment.\n# @PRE: ID is provided.\n# @POST: Returns success or error status.\n# @PARAM: id (str) - The ID of the environment to test.\n# @RETURN: dict - Success message or error.\n@router.post(\"/environments/{id}/test\")\nasync def test_environment_connection(\n id: str, config_manager: ConfigManager = Depends(get_config_manager)\n):\n with belief_scope(\"test_environment_connection\"):\n log.reason(\"Testing environment connection\", payload={\"id\": id})\n\n # Find environment\n env = next((e for e in config_manager.get_environments() if e.id == id), None)\n if not env:\n raise HTTPException(status_code=404, detail=f\"Environment {id} not found\")\n\n try:\n _validate_superset_connection_fast(env)\n\n log.reflect(\"Connection successful\", payload={\"id\": id})\n return {\"status\": \"success\", \"message\": \"Connection successful\"}\n except Exception as e:\n log.explore(\"Connection failed\", payload={\"id\": id}, error=str(e))\n return {\"status\": \"error\", \"message\": str(e)}\n\n\n# [/DEF:test_environment_connection:Function]\n\n\n# [DEF:get_logging_config:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves current logging configuration.\n# @PRE: Config manager is available.\n# @POST: Returns logging configuration.\n# @RETURN: LoggingConfigResponse - The current logging config.\n@router.get(\"/logging\", response_model=LoggingConfigResponse)\nasync def get_logging_config(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_logging_config\"):\n logging_config = config_manager.get_config().settings.logging\n return LoggingConfigResponse(\n level=logging_config.level,\n task_log_level=logging_config.task_log_level,\n enable_belief_state=logging_config.enable_belief_state,\n )\n\n\n# [/DEF:get_logging_config:Function]\n\n\n# [DEF:update_logging_config:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates logging configuration.\n# @PRE: New logging config is provided.\n# @POST: Logging configuration is updated and saved.\n# @PARAM: config (LoggingConfig) - The new logging configuration.\n# @RETURN: LoggingConfigResponse - The updated logging config.\n@router.patch(\"/logging\", response_model=LoggingConfigResponse)\nasync def update_logging_config(\n config: LoggingConfig,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_logging_config\"):\n log.reason(\"Updating logging config\", payload={\"level\": config.level, \"task_log_level\": config.task_log_level})\n\n # Get current settings and update logging config\n settings = config_manager.get_config().settings\n settings.logging = config\n config_manager.update_global_settings(settings)\n\n return LoggingConfigResponse(\n level=config.level,\n task_log_level=config.task_log_level,\n enable_belief_state=config.enable_belief_state,\n )\n\n\n# [/DEF:update_logging_config:Function]\n\n\n# [DEF:ConsolidatedSettingsResponse:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Response model for consolidated application settings.\nclass ConsolidatedSettingsResponse(BaseModel):\n environments: List[dict]\n connections: List[dict]\n llm: dict\n llm_providers: List[dict]\n logging: dict\n storage: dict\n notifications: dict = {}\n features: dict = {}\n\n\n# [/DEF:ConsolidatedSettingsResponse:Class]\n\n\n# [DEF:get_consolidated_settings:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Retrieves all settings categories in a single call.\n# @PRE: Config manager is available and the caller holds admin settings read permission.\n# @POST: Returns consolidated settings, provider metadata, and persisted notification payload in one stable response.\n# @SIDE_EFFECT: Opens one database session to read LLM providers and config-backed notification payload, then closes it.\n# @DATA_CONTRACT: Input[ConfigManager] -> Output[ConsolidatedSettingsResponse]\n# @RELATION: [DEPENDS_ON] ->[ConfigManager]\n# @RELATION: [DEPENDS_ON] ->[LLMProviderService]\n# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]\n# @RELATION: [DEPENDS_ON] ->[SessionLocal]\n# @RELATION: [DEPENDS_ON] ->[has_permission]\n# @RELATION: [DEPENDS_ON] ->[normalize_llm_settings]\n@router.get(\"/consolidated\", response_model=ConsolidatedSettingsResponse)\nasync def get_consolidated_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_consolidated_settings\"):\n log.reason(\"Fetching consolidated settings payload\")\n\n config = config_manager.get_config()\n\n from ...services.llm_provider import LLMProviderService\n from ...core.database import SessionLocal\n\n db = SessionLocal()\n notifications_payload = {}\n try:\n llm_service = LLMProviderService(db)\n providers = llm_service.get_all_providers()\n llm_providers_list = [\n {\n \"id\": p.id,\n \"provider_type\": p.provider_type,\n \"name\": p.name,\n \"base_url\": p.base_url,\n \"api_key\": \"********\",\n \"default_model\": p.default_model,\n \"is_active\": p.is_active,\n }\n for p in providers\n ]\n\n config_record = (\n db.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n )\n if config_record and isinstance(config_record.payload, dict):\n notifications_payload = (\n config_record.payload.get(\"notifications\", {}) or {}\n )\n finally:\n db.close()\n\n normalized_llm = normalize_llm_settings(config.settings.llm)\n\n response_payload = ConsolidatedSettingsResponse(\n environments=[env.dict() for env in config.environments],\n connections=config.settings.connections,\n llm=normalized_llm,\n llm_providers=llm_providers_list,\n logging=config.settings.logging.dict(),\n storage=config.settings.storage.dict(),\n notifications=notifications_payload,\n features=config.settings.features.model_dump(),\n )\n log.reflect(\n \"Consolidated settings payload assembled\",\n payload={\n \"environment_count\": len(response_payload.environments),\n \"provider_count\": len(response_payload.llm_providers),\n },\n )\n return response_payload\n\n\n# [/DEF:get_consolidated_settings:Function]\n\n\n# [DEF:update_consolidated_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Bulk update application settings from the consolidated view.\n# @PRE: User has admin permissions, config is valid.\n# @POST: Settings are updated and saved via ConfigManager.\n@router.patch(\"/consolidated\")\nasync def update_consolidated_settings(\n settings_patch: dict,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_consolidated_settings\"):\n log.reason(\"Applying consolidated settings patch\")\n\n current_config = config_manager.get_config()\n current_settings = current_config.settings\n\n # Update connections if provided\n if \"connections\" in settings_patch:\n current_settings.connections = settings_patch[\"connections\"]\n\n # Update LLM if provided\n if \"llm\" in settings_patch:\n current_settings.llm = normalize_llm_settings(settings_patch[\"llm\"])\n\n # Update Logging if provided\n if \"logging\" in settings_patch:\n current_settings.logging = LoggingConfig(**settings_patch[\"logging\"])\n\n # Update Storage if provided\n if \"storage\" in settings_patch:\n new_storage = StorageConfig(**settings_patch[\"storage\"])\n is_valid, message = config_manager.validate_path(new_storage.root_path)\n if not is_valid:\n raise HTTPException(status_code=400, detail=message)\n current_settings.storage = new_storage\n\n # Update Features if provided\n if \"features\" in settings_patch:\n from ...core.config_models import FeaturesConfig\n\n current_settings.features = FeaturesConfig(**settings_patch[\"features\"])\n\n if \"notifications\" in settings_patch:\n payload = config_manager.get_payload()\n payload[\"notifications\"] = settings_patch[\"notifications\"]\n config_manager.save_config(payload)\n\n config_manager.update_global_settings(current_settings)\n return {\"status\": \"success\", \"message\": \"Settings updated\"}\n\n\n# [/DEF:update_consolidated_settings:Function]\n\n\n# [DEF:get_validation_policies:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Lists all validation policies.\n# @RETURN: List[ValidationPolicyResponse] - List of policies.\n@router.get(\"/automation/policies\", response_model=List[ValidationPolicyResponse])\nasync def get_validation_policies(\n db: Session = Depends(get_db), _=Depends(has_permission(\"admin:settings\", \"READ\"))\n):\n with belief_scope(\"get_validation_policies\"):\n return db.query(ValidationPolicy).all()\n\n\n# [/DEF:get_validation_policies:Function]\n\n\n# [DEF:create_validation_policy:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Creates a new validation policy.\n# @PARAM: policy (ValidationPolicyCreate) - The policy data.\n# @RETURN: ValidationPolicyResponse - The created policy.\n@router.post(\"/automation/policies\", response_model=ValidationPolicyResponse)\nasync def create_validation_policy(\n policy: ValidationPolicyCreate,\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"create_validation_policy\"):\n db_policy = ValidationPolicy(**policy.dict())\n db.add(db_policy)\n db.commit()\n db.refresh(db_policy)\n return db_policy\n\n\n# [/DEF:create_validation_policy:Function]\n\n\n# [DEF:update_validation_policy:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates an existing validation policy.\n# @PARAM: id (str) - The ID of the policy to update.\n# @PARAM: policy (ValidationPolicyUpdate) - The updated policy data.\n# @RETURN: ValidationPolicyResponse - The updated policy.\n@router.patch(\"/automation/policies/{id}\", response_model=ValidationPolicyResponse)\nasync def update_validation_policy(\n id: str,\n policy: ValidationPolicyUpdate,\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_validation_policy\"):\n db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first()\n if not db_policy:\n raise HTTPException(status_code=404, detail=\"Policy not found\")\n\n update_data = policy.dict(exclude_unset=True)\n for key, value in update_data.items():\n setattr(db_policy, key, value)\n\n db.commit()\n db.refresh(db_policy)\n return db_policy\n\n\n# [/DEF:update_validation_policy:Function]\n\n\n# [DEF:delete_validation_policy:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Deletes a validation policy.\n# @PARAM: id (str) - The ID of the policy to delete.\n@router.delete(\"/automation/policies/{id}\")\nasync def delete_validation_policy(\n id: str,\n db: Session = Depends(get_db),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"delete_validation_policy\"):\n db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first()\n if not db_policy:\n raise HTTPException(status_code=404, detail=\"Policy not found\")\n\n db.delete(db_policy)\n db.commit()\n return {\"message\": \"Policy deleted\"}\n\n\n# [/DEF:delete_validation_policy:Function]\n\n# [/DEF:SettingsRouter:Module]\n" }, { "contract_id": "LoggingConfigResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/settings.py", - "start_line": 37, - "end_line": 47, + "start_line": 40, + "end_line": 50, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -22781,8 +22781,8 @@ "contract_id": "_validate_superset_connection_fast", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 67, - "end_line": 86, + "start_line": 70, + "end_line": 89, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -22819,8 +22819,8 @@ "contract_id": "get_settings", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 89, - "end_line": 111, + "start_line": 92, + "end_line": 114, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -22858,14 +22858,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves all application settings.\n# @PRE: Config manager is available.\n# @POST: Returns masked AppConfig.\n# @RETURN: AppConfig - The current configuration.\n@router.get(\"\", response_model=AppConfig)\nasync def get_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_settings\"):\n logger.info(\"[get_settings][Entry] Fetching all settings\")\n config = config_manager.get_config().copy(deep=True)\n config.settings.llm = normalize_llm_settings(config.settings.llm)\n # Mask passwords\n for env in config.environments:\n if env.password:\n env.password = \"********\"\n return config\n\n\n# [/DEF:get_settings:Function]\n" + "body": "# [DEF:get_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Retrieves all application settings.\n# @PRE: Config manager is available.\n# @POST: Returns masked AppConfig.\n# @RETURN: AppConfig - The current configuration.\n@router.get(\"\", response_model=AppConfig)\nasync def get_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_settings\"):\n log.reason(\"Fetching all settings\")\n config = config_manager.get_config().copy(deep=True)\n config.settings.llm = normalize_llm_settings(config.settings.llm)\n # Mask passwords\n for env in config.environments:\n if env.password:\n env.password = \"********\"\n return config\n\n\n# [/DEF:get_settings:Function]\n" }, { "contract_id": "get_features", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 114, - "end_line": 127, + "start_line": 117, + "end_line": 130, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -22903,8 +22903,8 @@ "contract_id": "get_storage_settings", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 153, - "end_line": 166, + "start_line": 156, + "end_line": 169, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -22928,8 +22928,8 @@ "contract_id": "update_storage_settings", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 169, - "end_line": 192, + "start_line": 172, + "end_line": 195, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -22970,8 +22970,8 @@ "contract_id": "test_environment_connection", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 319, - "end_line": 352, + "start_line": 318, + "end_line": 347, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23016,14 +23016,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_environment_connection:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Tests the connection to a Superset environment.\n# @PRE: ID is provided.\n# @POST: Returns success or error status.\n# @PARAM: id (str) - The ID of the environment to test.\n# @RETURN: dict - Success message or error.\n@router.post(\"/environments/{id}/test\")\nasync def test_environment_connection(\n id: str, config_manager: ConfigManager = Depends(get_config_manager)\n):\n with belief_scope(\"test_environment_connection\"):\n logger.info(f\"[test_environment_connection][Entry] Testing environment {id}\")\n\n # Find environment\n env = next((e for e in config_manager.get_environments() if e.id == id), None)\n if not env:\n raise HTTPException(status_code=404, detail=f\"Environment {id} not found\")\n\n try:\n _validate_superset_connection_fast(env)\n\n logger.info(\n f\"[test_environment_connection][Coherence:OK] Connection successful for {id}\"\n )\n return {\"status\": \"success\", \"message\": \"Connection successful\"}\n except Exception as e:\n logger.error(\n f\"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}\"\n )\n return {\"status\": \"error\", \"message\": str(e)}\n\n\n# [/DEF:test_environment_connection:Function]\n" + "body": "# [DEF:test_environment_connection:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Tests the connection to a Superset environment.\n# @PRE: ID is provided.\n# @POST: Returns success or error status.\n# @PARAM: id (str) - The ID of the environment to test.\n# @RETURN: dict - Success message or error.\n@router.post(\"/environments/{id}/test\")\nasync def test_environment_connection(\n id: str, config_manager: ConfigManager = Depends(get_config_manager)\n):\n with belief_scope(\"test_environment_connection\"):\n log.reason(\"Testing environment connection\", payload={\"id\": id})\n\n # Find environment\n env = next((e for e in config_manager.get_environments() if e.id == id), None)\n if not env:\n raise HTTPException(status_code=404, detail=f\"Environment {id} not found\")\n\n try:\n _validate_superset_connection_fast(env)\n\n log.reflect(\"Connection successful\", payload={\"id\": id})\n return {\"status\": \"success\", \"message\": \"Connection successful\"}\n except Exception as e:\n log.explore(\"Connection failed\", payload={\"id\": id}, error=str(e))\n return {\"status\": \"error\", \"message\": str(e)}\n\n\n# [/DEF:test_environment_connection:Function]\n" }, { "contract_id": "get_logging_config", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 355, - "end_line": 375, + "start_line": 350, + "end_line": 370, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23067,8 +23067,8 @@ "contract_id": "update_logging_config", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 378, - "end_line": 408, + "start_line": 373, + "end_line": 401, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23113,14 +23113,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:update_logging_config:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates logging configuration.\n# @PRE: New logging config is provided.\n# @POST: Logging configuration is updated and saved.\n# @PARAM: config (LoggingConfig) - The new logging configuration.\n# @RETURN: LoggingConfigResponse - The updated logging config.\n@router.patch(\"/logging\", response_model=LoggingConfigResponse)\nasync def update_logging_config(\n config: LoggingConfig,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_logging_config\"):\n logger.info(\n f\"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}\"\n )\n\n # Get current settings and update logging config\n settings = config_manager.get_config().settings\n settings.logging = config\n config_manager.update_global_settings(settings)\n\n return LoggingConfigResponse(\n level=config.level,\n task_log_level=config.task_log_level,\n enable_belief_state=config.enable_belief_state,\n )\n\n\n# [/DEF:update_logging_config:Function]\n" + "body": "# [DEF:update_logging_config:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Updates logging configuration.\n# @PRE: New logging config is provided.\n# @POST: Logging configuration is updated and saved.\n# @PARAM: config (LoggingConfig) - The new logging configuration.\n# @RETURN: LoggingConfigResponse - The updated logging config.\n@router.patch(\"/logging\", response_model=LoggingConfigResponse)\nasync def update_logging_config(\n config: LoggingConfig,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_logging_config\"):\n log.reason(\"Updating logging config\", payload={\"level\": config.level, \"task_log_level\": config.task_log_level})\n\n # Get current settings and update logging config\n settings = config_manager.get_config().settings\n settings.logging = config\n config_manager.update_global_settings(settings)\n\n return LoggingConfigResponse(\n level=config.level,\n task_log_level=config.task_log_level,\n enable_belief_state=config.enable_belief_state,\n )\n\n\n# [/DEF:update_logging_config:Function]\n" }, { "contract_id": "ConsolidatedSettingsResponse", "contract_type": "Class", "file_path": "backend/src/api/routes/settings.py", - "start_line": 411, - "end_line": 425, + "start_line": 404, + "end_line": 418, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -23136,8 +23136,8 @@ "contract_id": "get_consolidated_settings", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 428, - "end_line": 504, + "start_line": 421, + "end_line": 497, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -23300,14 +23300,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_consolidated_settings:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Retrieves all settings categories in a single call.\n# @PRE: Config manager is available and the caller holds admin settings read permission.\n# @POST: Returns consolidated settings, provider metadata, and persisted notification payload in one stable response.\n# @SIDE_EFFECT: Opens one database session to read LLM providers and config-backed notification payload, then closes it.\n# @DATA_CONTRACT: Input[ConfigManager] -> Output[ConsolidatedSettingsResponse]\n# @RELATION: [DEPENDS_ON] ->[ConfigManager]\n# @RELATION: [DEPENDS_ON] ->[LLMProviderService]\n# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]\n# @RELATION: [DEPENDS_ON] ->[SessionLocal]\n# @RELATION: [DEPENDS_ON] ->[has_permission]\n# @RELATION: [DEPENDS_ON] ->[normalize_llm_settings]\n@router.get(\"/consolidated\", response_model=ConsolidatedSettingsResponse)\nasync def get_consolidated_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_consolidated_settings\"):\n logger.reason(\"Fetching consolidated settings payload\")\n\n config = config_manager.get_config()\n\n from ...services.llm_provider import LLMProviderService\n from ...core.database import SessionLocal\n\n db = SessionLocal()\n notifications_payload = {}\n try:\n llm_service = LLMProviderService(db)\n providers = llm_service.get_all_providers()\n llm_providers_list = [\n {\n \"id\": p.id,\n \"provider_type\": p.provider_type,\n \"name\": p.name,\n \"base_url\": p.base_url,\n \"api_key\": \"********\",\n \"default_model\": p.default_model,\n \"is_active\": p.is_active,\n }\n for p in providers\n ]\n\n config_record = (\n db.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n )\n if config_record and isinstance(config_record.payload, dict):\n notifications_payload = (\n config_record.payload.get(\"notifications\", {}) or {}\n )\n finally:\n db.close()\n\n normalized_llm = normalize_llm_settings(config.settings.llm)\n\n response_payload = ConsolidatedSettingsResponse(\n environments=[env.dict() for env in config.environments],\n connections=config.settings.connections,\n llm=normalized_llm,\n llm_providers=llm_providers_list,\n logging=config.settings.logging.dict(),\n storage=config.settings.storage.dict(),\n notifications=notifications_payload,\n features=config.settings.features.model_dump(),\n )\n logger.reflect(\n \"Consolidated settings payload assembled\",\n extra={\n \"environment_count\": len(response_payload.environments),\n \"provider_count\": len(response_payload.llm_providers),\n },\n )\n return response_payload\n\n\n# [/DEF:get_consolidated_settings:Function]\n" + "body": "# [DEF:get_consolidated_settings:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Retrieves all settings categories in a single call.\n# @PRE: Config manager is available and the caller holds admin settings read permission.\n# @POST: Returns consolidated settings, provider metadata, and persisted notification payload in one stable response.\n# @SIDE_EFFECT: Opens one database session to read LLM providers and config-backed notification payload, then closes it.\n# @DATA_CONTRACT: Input[ConfigManager] -> Output[ConsolidatedSettingsResponse]\n# @RELATION: [DEPENDS_ON] ->[ConfigManager]\n# @RELATION: [DEPENDS_ON] ->[LLMProviderService]\n# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]\n# @RELATION: [DEPENDS_ON] ->[SessionLocal]\n# @RELATION: [DEPENDS_ON] ->[has_permission]\n# @RELATION: [DEPENDS_ON] ->[normalize_llm_settings]\n@router.get(\"/consolidated\", response_model=ConsolidatedSettingsResponse)\nasync def get_consolidated_settings(\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"READ\")),\n):\n with belief_scope(\"get_consolidated_settings\"):\n log.reason(\"Fetching consolidated settings payload\")\n\n config = config_manager.get_config()\n\n from ...services.llm_provider import LLMProviderService\n from ...core.database import SessionLocal\n\n db = SessionLocal()\n notifications_payload = {}\n try:\n llm_service = LLMProviderService(db)\n providers = llm_service.get_all_providers()\n llm_providers_list = [\n {\n \"id\": p.id,\n \"provider_type\": p.provider_type,\n \"name\": p.name,\n \"base_url\": p.base_url,\n \"api_key\": \"********\",\n \"default_model\": p.default_model,\n \"is_active\": p.is_active,\n }\n for p in providers\n ]\n\n config_record = (\n db.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n )\n if config_record and isinstance(config_record.payload, dict):\n notifications_payload = (\n config_record.payload.get(\"notifications\", {}) or {}\n )\n finally:\n db.close()\n\n normalized_llm = normalize_llm_settings(config.settings.llm)\n\n response_payload = ConsolidatedSettingsResponse(\n environments=[env.dict() for env in config.environments],\n connections=config.settings.connections,\n llm=normalized_llm,\n llm_providers=llm_providers_list,\n logging=config.settings.logging.dict(),\n storage=config.settings.storage.dict(),\n notifications=notifications_payload,\n features=config.settings.features.model_dump(),\n )\n log.reflect(\n \"Consolidated settings payload assembled\",\n payload={\n \"environment_count\": len(response_payload.environments),\n \"provider_count\": len(response_payload.llm_providers),\n },\n )\n return response_payload\n\n\n# [/DEF:get_consolidated_settings:Function]\n" }, { "contract_id": "update_consolidated_settings", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 507, - "end_line": 561, + "start_line": 500, + "end_line": 552, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23338,14 +23338,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:update_consolidated_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Bulk update application settings from the consolidated view.\n# @PRE: User has admin permissions, config is valid.\n# @POST: Settings are updated and saved via ConfigManager.\n@router.patch(\"/consolidated\")\nasync def update_consolidated_settings(\n settings_patch: dict,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_consolidated_settings\"):\n logger.info(\n \"[update_consolidated_settings][Entry] Applying consolidated settings patch\"\n )\n\n current_config = config_manager.get_config()\n current_settings = current_config.settings\n\n # Update connections if provided\n if \"connections\" in settings_patch:\n current_settings.connections = settings_patch[\"connections\"]\n\n # Update LLM if provided\n if \"llm\" in settings_patch:\n current_settings.llm = normalize_llm_settings(settings_patch[\"llm\"])\n\n # Update Logging if provided\n if \"logging\" in settings_patch:\n current_settings.logging = LoggingConfig(**settings_patch[\"logging\"])\n\n # Update Storage if provided\n if \"storage\" in settings_patch:\n new_storage = StorageConfig(**settings_patch[\"storage\"])\n is_valid, message = config_manager.validate_path(new_storage.root_path)\n if not is_valid:\n raise HTTPException(status_code=400, detail=message)\n current_settings.storage = new_storage\n\n # Update Features if provided\n if \"features\" in settings_patch:\n from ...core.config_models import FeaturesConfig\n\n current_settings.features = FeaturesConfig(**settings_patch[\"features\"])\n\n if \"notifications\" in settings_patch:\n payload = config_manager.get_payload()\n payload[\"notifications\"] = settings_patch[\"notifications\"]\n config_manager.save_config(payload)\n\n config_manager.update_global_settings(current_settings)\n return {\"status\": \"success\", \"message\": \"Settings updated\"}\n\n\n# [/DEF:update_consolidated_settings:Function]\n" + "body": "# [DEF:update_consolidated_settings:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Bulk update application settings from the consolidated view.\n# @PRE: User has admin permissions, config is valid.\n# @POST: Settings are updated and saved via ConfigManager.\n@router.patch(\"/consolidated\")\nasync def update_consolidated_settings(\n settings_patch: dict,\n config_manager: ConfigManager = Depends(get_config_manager),\n _=Depends(has_permission(\"admin:settings\", \"WRITE\")),\n):\n with belief_scope(\"update_consolidated_settings\"):\n log.reason(\"Applying consolidated settings patch\")\n\n current_config = config_manager.get_config()\n current_settings = current_config.settings\n\n # Update connections if provided\n if \"connections\" in settings_patch:\n current_settings.connections = settings_patch[\"connections\"]\n\n # Update LLM if provided\n if \"llm\" in settings_patch:\n current_settings.llm = normalize_llm_settings(settings_patch[\"llm\"])\n\n # Update Logging if provided\n if \"logging\" in settings_patch:\n current_settings.logging = LoggingConfig(**settings_patch[\"logging\"])\n\n # Update Storage if provided\n if \"storage\" in settings_patch:\n new_storage = StorageConfig(**settings_patch[\"storage\"])\n is_valid, message = config_manager.validate_path(new_storage.root_path)\n if not is_valid:\n raise HTTPException(status_code=400, detail=message)\n current_settings.storage = new_storage\n\n # Update Features if provided\n if \"features\" in settings_patch:\n from ...core.config_models import FeaturesConfig\n\n current_settings.features = FeaturesConfig(**settings_patch[\"features\"])\n\n if \"notifications\" in settings_patch:\n payload = config_manager.get_payload()\n payload[\"notifications\"] = settings_patch[\"notifications\"]\n config_manager.save_config(payload)\n\n config_manager.update_global_settings(current_settings)\n return {\"status\": \"success\", \"message\": \"Settings updated\"}\n\n\n# [/DEF:update_consolidated_settings:Function]\n" }, { "contract_id": "get_validation_policies", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 564, - "end_line": 576, + "start_line": 555, + "end_line": 567, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23369,8 +23369,8 @@ "contract_id": "create_validation_policy", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 579, - "end_line": 598, + "start_line": 570, + "end_line": 589, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23401,8 +23401,8 @@ "contract_id": "update_validation_policy", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 601, - "end_line": 628, + "start_line": 592, + "end_line": 619, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -23433,8 +23433,8 @@ "contract_id": "delete_validation_policy", "contract_type": "Function", "file_path": "backend/src/api/routes/settings.py", - "start_line": 631, - "end_line": 651, + "start_line": 622, + "end_line": 642, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -24052,7 +24052,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_correction_routes.py", "start_line": 1, - "end_line": 105, + "end_line": 106, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24101,14 +24101,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateCorrectionRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,corrections]\n# @BRIEF Term correction submission endpoints for applying user feedback to dictionary entries.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE DB session initialized. User authenticated with translate.dictionary.edit permissions.\n# @POST Term corrections submitted and applied to dictionary with conflict detection.\n# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB based on correction submissions.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission\nfrom ....plugins.translate.dictionary import DictionaryManager\nfrom ....schemas.translate import (\n TermCorrectionSubmit,\n TermCorrectionBulkSubmit,\n CorrectionSubmitResponse,\n)\n\nfrom ._router import router\n\n\n# ============================================================\n# Corrections\n# ============================================================\n\n# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit a single term correction from a run result review.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST Correction applied with conflict detection. Result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections\")\nasync def submit_correction(\n payload: TermCorrectionSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit a term correction from a run result review.\"\"\"\n logger.info(f\"[translate_routes][submit_correction] User: {current_user.username}\")\n if not payload.dictionary_id:\n raise HTTPException(status_code=422, detail=\"dictionary_id is required\")\n try:\n result = DictionaryManager.submit_correction(\n db,\n dict_id=payload.dictionary_id,\n source_term=payload.source_term,\n incorrect_target_term=payload.incorrect_target_term,\n corrected_target_term=payload.corrected_target_term,\n origin_run_id=payload.origin_run_id,\n origin_row_key=payload.origin_row_key,\n origin_user_id=current_user.username,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_correction\n\n\n# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit multiple term corrections atomically with full conflict reporting.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST All corrections applied or none with conflict list returned.\n# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections/bulk\")\nasync def submit_bulk_corrections(\n payload: TermCorrectionBulkSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit multiple term corrections atomically.\"\"\"\n logger.info(f\"[translate_routes][submit_bulk_corrections] User: {current_user.username}\")\n try:\n corr_list = []\n for c in payload.corrections:\n corr_list.append({\n \"source_term\": c.source_term,\n \"incorrect_target_term\": c.incorrect_target_term,\n \"corrected_target_term\": c.corrected_target_term,\n \"origin_run_id\": c.origin_run_id,\n \"origin_row_key\": c.origin_row_key,\n })\n result = DictionaryManager.submit_bulk_corrections(\n db,\n dict_id=payload.dictionary_id,\n corrections=corr_list,\n origin_user_id=current_user.username,\n )\n if result.get(\"status\") == \"conflicts\":\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=result)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_bulk_corrections\n\n# #endregion TranslateCorrectionRoutesModule\n" + "body": "# #region TranslateCorrectionRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,corrections]\n# @BRIEF Term correction submission endpoints for applying user feedback to dictionary entries.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE DB session initialized. User authenticated with translate.dictionary.edit permissions.\n# @POST Term corrections submitted and applied to dictionary with conflict detection.\n# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB based on correction submissions.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission\nfrom ....plugins.translate.dictionary import DictionaryManager\nfrom ....schemas.translate import (\n TermCorrectionSubmit,\n TermCorrectionBulkSubmit,\n CorrectionSubmitResponse,\n)\n\nfrom ._router import router\n\nlog = MarkerLogger(\"TranslateCorrectionRoutes\")\n\n# ============================================================\n# Corrections\n# ============================================================\n\n# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit a single term correction from a run result review.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST Correction applied with conflict detection. Result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections\")\nasync def submit_correction(\n payload: TermCorrectionSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit a term correction from a run result review.\"\"\"\n log.reason(\"Correction request\", payload={\"user\": current_user.username})\n if not payload.dictionary_id:\n raise HTTPException(status_code=422, detail=\"dictionary_id is required\")\n try:\n result = DictionaryManager.submit_correction(\n db,\n dict_id=payload.dictionary_id,\n source_term=payload.source_term,\n incorrect_target_term=payload.incorrect_target_term,\n corrected_target_term=payload.corrected_target_term,\n origin_run_id=payload.origin_run_id,\n origin_row_key=payload.origin_row_key,\n origin_user_id=current_user.username,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_correction\n\n\n# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit multiple term corrections atomically with full conflict reporting.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST All corrections applied or none with conflict list returned.\n# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections/bulk\")\nasync def submit_bulk_corrections(\n payload: TermCorrectionBulkSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit multiple term corrections atomically.\"\"\"\n log.reason(\"Bulk correction request\", payload={\"user\": current_user.username})\n try:\n corr_list = []\n for c in payload.corrections:\n corr_list.append({\n \"source_term\": c.source_term,\n \"incorrect_target_term\": c.incorrect_target_term,\n \"corrected_target_term\": c.corrected_target_term,\n \"origin_run_id\": c.origin_run_id,\n \"origin_row_key\": c.origin_row_key,\n })\n result = DictionaryManager.submit_bulk_corrections(\n db,\n dict_id=payload.dictionary_id,\n corrections=corr_list,\n origin_user_id=current_user.username,\n )\n if result.get(\"status\") == \"conflicts\":\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=result)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_bulk_corrections\n\n# #endregion TranslateCorrectionRoutesModule\n" }, { "contract_id": "submit_correction", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_correction_routes.py", - "start_line": 33, - "end_line": 64, + "start_line": 34, + "end_line": 65, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24144,14 +24144,14 @@ } ], "anchor_syntax": "region", - "body": "# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit a single term correction from a run result review.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST Correction applied with conflict detection. Result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections\")\nasync def submit_correction(\n payload: TermCorrectionSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit a term correction from a run result review.\"\"\"\n logger.info(f\"[translate_routes][submit_correction] User: {current_user.username}\")\n if not payload.dictionary_id:\n raise HTTPException(status_code=422, detail=\"dictionary_id is required\")\n try:\n result = DictionaryManager.submit_correction(\n db,\n dict_id=payload.dictionary_id,\n source_term=payload.source_term,\n incorrect_target_term=payload.incorrect_target_term,\n corrected_target_term=payload.corrected_target_term,\n origin_run_id=payload.origin_run_id,\n origin_row_key=payload.origin_row_key,\n origin_user_id=current_user.username,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_correction\n" + "body": "# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit a single term correction from a run result review.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST Correction applied with conflict detection. Result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections\")\nasync def submit_correction(\n payload: TermCorrectionSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit a term correction from a run result review.\"\"\"\n log.reason(\"Correction request\", payload={\"user\": current_user.username})\n if not payload.dictionary_id:\n raise HTTPException(status_code=422, detail=\"dictionary_id is required\")\n try:\n result = DictionaryManager.submit_correction(\n db,\n dict_id=payload.dictionary_id,\n source_term=payload.source_term,\n incorrect_target_term=payload.incorrect_target_term,\n corrected_target_term=payload.corrected_target_term,\n origin_run_id=payload.origin_run_id,\n origin_row_key=payload.origin_row_key,\n origin_user_id=current_user.username,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_correction\n" }, { "contract_id": "submit_bulk_corrections", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_correction_routes.py", - "start_line": 67, - "end_line": 103, + "start_line": 68, + "end_line": 104, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24187,14 +24187,14 @@ } ], "anchor_syntax": "region", - "body": "# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit multiple term corrections atomically with full conflict reporting.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST All corrections applied or none with conflict list returned.\n# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections/bulk\")\nasync def submit_bulk_corrections(\n payload: TermCorrectionBulkSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit multiple term corrections atomically.\"\"\"\n logger.info(f\"[translate_routes][submit_bulk_corrections] User: {current_user.username}\")\n try:\n corr_list = []\n for c in payload.corrections:\n corr_list.append({\n \"source_term\": c.source_term,\n \"incorrect_target_term\": c.incorrect_target_term,\n \"corrected_target_term\": c.corrected_target_term,\n \"origin_run_id\": c.origin_run_id,\n \"origin_row_key\": c.origin_row_key,\n })\n result = DictionaryManager.submit_bulk_corrections(\n db,\n dict_id=payload.dictionary_id,\n corrections=corr_list,\n origin_user_id=current_user.username,\n )\n if result.get(\"status\") == \"conflicts\":\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=result)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_bulk_corrections\n" + "body": "# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections]\n# @BRIEF Submit multiple term corrections atomically with full conflict reporting.\n# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided.\n# @POST All corrections applied or none with conflict list returned.\n# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/corrections/bulk\")\nasync def submit_bulk_corrections(\n payload: TermCorrectionBulkSubmit,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Submit multiple term corrections atomically.\"\"\"\n log.reason(\"Bulk correction request\", payload={\"user\": current_user.username})\n try:\n corr_list = []\n for c in payload.corrections:\n corr_list.append({\n \"source_term\": c.source_term,\n \"incorrect_target_term\": c.incorrect_target_term,\n \"corrected_target_term\": c.corrected_target_term,\n \"origin_run_id\": c.origin_run_id,\n \"origin_row_key\": c.origin_row_key,\n })\n result = DictionaryManager.submit_bulk_corrections(\n db,\n dict_id=payload.dictionary_id,\n corrections=corr_list,\n origin_user_id=current_user.username,\n )\n if result.get(\"status\") == \"conflicts\":\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=result)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion submit_bulk_corrections\n" }, { "contract_id": "TranslateDictionaryRoutesModule", "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", "start_line": 1, - "end_line": 380, + "end_line": 383, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24243,14 +24243,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateDictionaryRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,dictionaries]\n# @BRIEF Terminology Dictionary CRUD, entries management, and import routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE DB session initialized. User authenticated with translate.dictionary permissions.\n# @POST Dictionaries and entries CRUD completed. Import executed with conflict resolution.\n# @SIDE_EFFECT Reads/writes TerminologyDictionary and DictionaryEntry records to DB.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission\nfrom ....plugins.translate.dictionary import DictionaryManager\nfrom ....schemas.translate import (\n DictionaryCreate,\n DictionaryEntryCreate,\n DictionaryEntryResponse,\n DictionaryImport,\n DictionaryResponse,\n)\n\nfrom ._router import router\nfrom ._helpers import _dict_to_response, _get_dictionary_entry_counts\n\n\n# ============================================================\n# Terminology Dictionaries\n# ============================================================\n\n# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF List all terminology dictionaries with pagination and entry counts.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns paginated list of dictionaries with total count.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries\")\nasync def list_dictionaries(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all terminology dictionaries.\"\"\"\n with belief_scope(\"list_dictionaries\"):\n logger.reason(f\"User: {current_user.username}\")\n dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size)\n dict_ids = [d.id for d in dicts]\n counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {}\n items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]\n logger.reflect(\"Dictionaries listed\", {\"total\": total, \"returned\": len(items)})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n# #endregion list_dictionaries\n\n\n# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Get a single terminology dictionary by ID with entry count.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns the dictionary with entry_count or 404.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}\")\nasync def get_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get a terminology dictionary by ID.\"\"\"\n with belief_scope(\"get_dictionary\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n d = DictionaryManager.get_dictionary(db, dictionary_id)\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n logger.reflect(\"Dictionary found\", {\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_dictionary\n\n\n# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Create a new terminology dictionary.\n# @PRE User has translate.dictionary.create permission. Payload validated.\n# @POST Dictionary created and returned with zero entries.\n# @SIDE_EFFECT Creates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries\", status_code=status.HTTP_201_CREATED)\nasync def create_dictionary(\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"CREATE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Create a new terminology dictionary.\"\"\"\n with belief_scope(\"create_dictionary\"):\n logger.reason(f\"User: {current_user.username}\")\n d = DictionaryManager.create_dictionary(\n db,\n name=payload.name,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n created_by=current_user.username,\n description=payload.description,\n is_active=payload.is_active,\n )\n logger.reflect(\"Dictionary created\", {\"id\": d.id})\n return _dict_to_response(d, 0)\n# #endregion create_dictionary\n\n\n# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Update an existing terminology dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Dictionary updated and returned with current entry count.\n# @SIDE_EFFECT Updates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}\")\nasync def update_dictionary(\n dictionary_id: str,\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update a terminology dictionary.\"\"\"\n with belief_scope(\"update_dictionary\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n d = DictionaryManager.update_dictionary(\n db,\n dict_id=dictionary_id,\n name=payload.name,\n description=payload.description,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n is_active=payload.is_active,\n )\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n logger.reflect(\"Dictionary updated\", {\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_dictionary\n\n\n# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.\n# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use.\n# @POST Dictionary is deleted or 409 if attached to active jobs.\n# @SIDE_EFFECT Deletes TerminologyDictionary record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"DELETE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a terminology dictionary.\"\"\"\n with belief_scope(\"delete_dictionary\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n DictionaryManager.delete_dictionary(db, dictionary_id)\n logger.reflect(\"Dictionary deleted\", {\"id\": dictionary_id})\n return None\n except ValueError as e:\n if \"active/scheduled\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary\n\n\n# ============================================================\n# Dictionary Entries CRUD\n# ============================================================\n\n# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF List entries for a dictionary with pagination.\n# @PRE User has translate.dictionary.view permission. Dictionary exists.\n# @POST Returns paginated list of entries.\n# @SIDE_EFFECT Reads DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}/entries\")\nasync def list_dictionary_entries(\n dictionary_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List entries for a dictionary.\"\"\"\n with belief_scope(\"list_dictionary_entries\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)\n items = [\n {\n \"id\": e.id,\n \"dictionary_id\": e.dictionary_id,\n \"source_term\": e.source_term,\n \"source_term_normalized\": e.source_term_normalized,\n \"target_term\": e.target_term,\n \"context_notes\": e.context_notes,\n \"created_at\": e.created_at,\n \"updated_at\": e.updated_at,\n }\n for e in entries\n ]\n logger.reflect(\"Entries listed\", {\"dict_id\": dictionary_id, \"total\": total})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion list_dictionary_entries\n\n\n# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Add a new entry to a dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Entry created and returned.\n# @SIDE_EFFECT Creates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/entries\", status_code=status.HTTP_201_CREATED)\nasync def add_dictionary_entry(\n dictionary_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Add a new entry to a dictionary.\"\"\"\n with belief_scope(\"add_dictionary_entry\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n entry = DictionaryManager.add_entry(\n db, dictionary_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n logger.reflect(\"Entry added\", {\"entry_id\": entry.id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion add_dictionary_entry\n\n\n# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Update an existing dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry updated and returned.\n# @SIDE_EFFECT Updates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}/entries/{entry_id}\")\nasync def edit_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update an existing dictionary entry.\"\"\"\n with belief_scope(\"edit_dictionary_entry\"):\n logger.reason(f\"Entry: {entry_id}, User: {current_user.username}\")\n try:\n entry = DictionaryManager.edit_entry(\n db, entry_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n logger.reflect(\"Entry updated\", {\"entry_id\": entry_id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion edit_dictionary_entry\n\n\n# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Delete a dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry is deleted.\n# @SIDE_EFFECT Deletes DictionaryEntry record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}/entries/{entry_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a dictionary entry.\"\"\"\n with belief_scope(\"delete_dictionary_entry\"):\n logger.reason(f\"Entry: {entry_id}, User: {current_user.username}\")\n try:\n DictionaryManager.delete_entry(db, entry_id)\n logger.reflect(\"Entry deleted\", {\"entry_id\": entry_id})\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary_entry\n\n\n# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import]\n# @BRIEF Import entries into a terminology dictionary from CSV/TSV content.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV.\n# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/import\")\nasync def import_dictionary_entries(\n dictionary_id: str,\n payload: DictionaryImport,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Import entries into a terminology dictionary from CSV/TSV content.\"\"\"\n with belief_scope(\"import_dictionary_entries\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n\n if payload.on_conflict not in (\"overwrite\", \"keep_existing\", \"cancel\"):\n raise HTTPException(\n status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,\n detail=\"on_conflict must be 'overwrite', 'keep_existing', or 'cancel'\",\n )\n\n try:\n result = DictionaryManager.import_entries(\n db,\n dict_id=dictionary_id,\n content=payload.content,\n delimiter=payload.delimiter,\n on_conflict=payload.on_conflict,\n preview_only=payload.preview_only,\n )\n logger.reflect(\"Import completed\", {\n \"dict_id\": dictionary_id,\n \"total\": result[\"total\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion import_dictionary_entries\n\n# #endregion TranslateDictionaryRoutesModule\n" + "body": "# #region TranslateDictionaryRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,dictionaries]\n# @BRIEF Terminology Dictionary CRUD, entries management, and import routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE DB session initialized. User authenticated with translate.dictionary permissions.\n# @POST Dictionaries and entries CRUD completed. Import executed with conflict resolution.\n# @SIDE_EFFECT Reads/writes TerminologyDictionary and DictionaryEntry records to DB.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import belief_scope\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\n\nlog = MarkerLogger(\"TranslateDictRoutes\")\nfrom ....dependencies import get_current_user, has_permission\nfrom ....plugins.translate.dictionary import DictionaryManager\nfrom ....schemas.translate import (\n DictionaryCreate,\n DictionaryEntryCreate,\n DictionaryEntryResponse,\n DictionaryImport,\n DictionaryResponse,\n)\n\nfrom ._router import router\nfrom ._helpers import _dict_to_response, _get_dictionary_entry_counts\n\n\n# ============================================================\n# Terminology Dictionaries\n# ============================================================\n\n# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF List all terminology dictionaries with pagination and entry counts.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns paginated list of dictionaries with total count.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries\")\nasync def list_dictionaries(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all terminology dictionaries.\"\"\"\n with belief_scope(\"list_dictionaries\"):\n log.reason(\"List dictionaries\", payload={\"user\": current_user.username})\n dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size)\n dict_ids = [d.id for d in dicts]\n counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {}\n items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]\n log.reflect(\"Dictionaries listed\", payload={\"total\": total, \"returned\": len(items)})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n# #endregion list_dictionaries\n\n\n# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Get a single terminology dictionary by ID with entry count.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns the dictionary with entry_count or 404.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}\")\nasync def get_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get a terminology dictionary by ID.\"\"\"\n with belief_scope(\"get_dictionary\"):\n log.reason(\"Get dictionary\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n d = DictionaryManager.get_dictionary(db, dictionary_id)\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n log.reflect(\"Dictionary found\", payload={\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_dictionary\n\n\n# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Create a new terminology dictionary.\n# @PRE User has translate.dictionary.create permission. Payload validated.\n# @POST Dictionary created and returned with zero entries.\n# @SIDE_EFFECT Creates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries\", status_code=status.HTTP_201_CREATED)\nasync def create_dictionary(\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"CREATE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Create a new terminology dictionary.\"\"\"\n with belief_scope(\"create_dictionary\"):\n log.reason(\"Create dictionary\", payload={\"user\": current_user.username})\n d = DictionaryManager.create_dictionary(\n db,\n name=payload.name,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n created_by=current_user.username,\n description=payload.description,\n is_active=payload.is_active,\n )\n log.reflect(\"Dictionary created\", payload={\"id\": d.id})\n return _dict_to_response(d, 0)\n# #endregion create_dictionary\n\n\n# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Update an existing terminology dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Dictionary updated and returned with current entry count.\n# @SIDE_EFFECT Updates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}\")\nasync def update_dictionary(\n dictionary_id: str,\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update a terminology dictionary.\"\"\"\n with belief_scope(\"update_dictionary\"):\n log.reason(\"Update dictionary\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n d = DictionaryManager.update_dictionary(\n db,\n dict_id=dictionary_id,\n name=payload.name,\n description=payload.description,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n is_active=payload.is_active,\n )\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n log.reflect(\"Dictionary updated\", payload={\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_dictionary\n\n\n# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.\n# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use.\n# @POST Dictionary is deleted or 409 if attached to active jobs.\n# @SIDE_EFFECT Deletes TerminologyDictionary record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"DELETE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a terminology dictionary.\"\"\"\n with belief_scope(\"delete_dictionary\"):\n log.reason(\"Delete dictionary\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n DictionaryManager.delete_dictionary(db, dictionary_id)\n log.reflect(\"Dictionary deleted\", payload={\"id\": dictionary_id})\n return None\n except ValueError as e:\n if \"active/scheduled\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary\n\n\n# ============================================================\n# Dictionary Entries CRUD\n# ============================================================\n\n# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF List entries for a dictionary with pagination.\n# @PRE User has translate.dictionary.view permission. Dictionary exists.\n# @POST Returns paginated list of entries.\n# @SIDE_EFFECT Reads DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}/entries\")\nasync def list_dictionary_entries(\n dictionary_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List entries for a dictionary.\"\"\"\n with belief_scope(\"list_dictionary_entries\"):\n log.reason(\"List dictionary entries\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)\n items = [\n {\n \"id\": e.id,\n \"dictionary_id\": e.dictionary_id,\n \"source_term\": e.source_term,\n \"source_term_normalized\": e.source_term_normalized,\n \"target_term\": e.target_term,\n \"context_notes\": e.context_notes,\n \"created_at\": e.created_at,\n \"updated_at\": e.updated_at,\n }\n for e in entries\n ]\n log.reflect(\"Entries listed\", payload={\"dict_id\": dictionary_id, \"total\": total})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion list_dictionary_entries\n\n\n# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Add a new entry to a dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Entry created and returned.\n# @SIDE_EFFECT Creates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/entries\", status_code=status.HTTP_201_CREATED)\nasync def add_dictionary_entry(\n dictionary_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Add a new entry to a dictionary.\"\"\"\n with belief_scope(\"add_dictionary_entry\"):\n log.reason(\"Add dictionary entry\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n entry = DictionaryManager.add_entry(\n db, dictionary_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n log.reflect(\"Entry added\", payload={\"entry_id\": entry.id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion add_dictionary_entry\n\n\n# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Update an existing dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry updated and returned.\n# @SIDE_EFFECT Updates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}/entries/{entry_id}\")\nasync def edit_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update an existing dictionary entry.\"\"\"\n with belief_scope(\"edit_dictionary_entry\"):\n log.reason(\"Edit dictionary entry\", payload={\"entry_id\": entry_id, \"user\": current_user.username})\n try:\n entry = DictionaryManager.edit_entry(\n db, entry_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n log.reflect(\"Entry updated\", payload={\"entry_id\": entry_id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion edit_dictionary_entry\n\n\n# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Delete a dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry is deleted.\n# @SIDE_EFFECT Deletes DictionaryEntry record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}/entries/{entry_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a dictionary entry.\"\"\"\n with belief_scope(\"delete_dictionary_entry\"):\n log.reason(\"Delete dictionary entry\", payload={\"entry_id\": entry_id, \"user\": current_user.username})\n try:\n DictionaryManager.delete_entry(db, entry_id)\n log.reflect(\"Entry deleted\", payload={\"entry_id\": entry_id})\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary_entry\n\n\n# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import]\n# @BRIEF Import entries into a terminology dictionary from CSV/TSV content.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV.\n# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/import\")\nasync def import_dictionary_entries(\n dictionary_id: str,\n payload: DictionaryImport,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Import entries into a terminology dictionary from CSV/TSV content.\"\"\"\n with belief_scope(\"import_dictionary_entries\"):\n log.reason(\"Import dictionary entries\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n\n if payload.on_conflict not in (\"overwrite\", \"keep_existing\", \"cancel\"):\n raise HTTPException(\n status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,\n detail=\"on_conflict must be 'overwrite', 'keep_existing', or 'cancel'\",\n )\n\n try:\n result = DictionaryManager.import_entries(\n db,\n dict_id=dictionary_id,\n content=payload.content,\n delimiter=payload.delimiter,\n on_conflict=payload.on_conflict,\n preview_only=payload.preview_only,\n )\n log.reflect(\"Import completed\", payload={\n \"dict_id\": dictionary_id,\n \"total\": result[\"total\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion import_dictionary_entries\n\n# #endregion TranslateDictionaryRoutesModule\n" }, { "contract_id": "list_dictionaries", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 36, - "end_line": 59, + "start_line": 39, + "end_line": 62, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24286,14 +24286,14 @@ } ], "anchor_syntax": "region", - "body": "# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF List all terminology dictionaries with pagination and entry counts.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns paginated list of dictionaries with total count.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries\")\nasync def list_dictionaries(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all terminology dictionaries.\"\"\"\n with belief_scope(\"list_dictionaries\"):\n logger.reason(f\"User: {current_user.username}\")\n dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size)\n dict_ids = [d.id for d in dicts]\n counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {}\n items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]\n logger.reflect(\"Dictionaries listed\", {\"total\": total, \"returned\": len(items)})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n# #endregion list_dictionaries\n" + "body": "# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF List all terminology dictionaries with pagination and entry counts.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns paginated list of dictionaries with total count.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries\")\nasync def list_dictionaries(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all terminology dictionaries.\"\"\"\n with belief_scope(\"list_dictionaries\"):\n log.reason(\"List dictionaries\", payload={\"user\": current_user.username})\n dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size)\n dict_ids = [d.id for d in dicts]\n counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {}\n items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts]\n log.reflect(\"Dictionaries listed\", payload={\"total\": total, \"returned\": len(items)})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n# #endregion list_dictionaries\n" }, { "contract_id": "get_dictionary", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 62, - "end_line": 86, + "start_line": 65, + "end_line": 89, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24329,14 +24329,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Get a single terminology dictionary by ID with entry count.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns the dictionary with entry_count or 404.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}\")\nasync def get_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get a terminology dictionary by ID.\"\"\"\n with belief_scope(\"get_dictionary\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n d = DictionaryManager.get_dictionary(db, dictionary_id)\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n logger.reflect(\"Dictionary found\", {\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_dictionary\n" + "body": "# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Get a single terminology dictionary by ID with entry count.\n# @PRE User has translate.dictionary.view permission.\n# @POST Returns the dictionary with entry_count or 404.\n# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}\")\nasync def get_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get a terminology dictionary by ID.\"\"\"\n with belief_scope(\"get_dictionary\"):\n log.reason(\"Get dictionary\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n d = DictionaryManager.get_dictionary(db, dictionary_id)\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n log.reflect(\"Dictionary found\", payload={\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_dictionary\n" }, { "contract_id": "create_dictionary", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 89, - "end_line": 116, + "start_line": 92, + "end_line": 119, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24372,14 +24372,14 @@ } ], "anchor_syntax": "region", - "body": "# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Create a new terminology dictionary.\n# @PRE User has translate.dictionary.create permission. Payload validated.\n# @POST Dictionary created and returned with zero entries.\n# @SIDE_EFFECT Creates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries\", status_code=status.HTTP_201_CREATED)\nasync def create_dictionary(\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"CREATE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Create a new terminology dictionary.\"\"\"\n with belief_scope(\"create_dictionary\"):\n logger.reason(f\"User: {current_user.username}\")\n d = DictionaryManager.create_dictionary(\n db,\n name=payload.name,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n created_by=current_user.username,\n description=payload.description,\n is_active=payload.is_active,\n )\n logger.reflect(\"Dictionary created\", {\"id\": d.id})\n return _dict_to_response(d, 0)\n# #endregion create_dictionary\n" + "body": "# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Create a new terminology dictionary.\n# @PRE User has translate.dictionary.create permission. Payload validated.\n# @POST Dictionary created and returned with zero entries.\n# @SIDE_EFFECT Creates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries\", status_code=status.HTTP_201_CREATED)\nasync def create_dictionary(\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"CREATE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Create a new terminology dictionary.\"\"\"\n with belief_scope(\"create_dictionary\"):\n log.reason(\"Create dictionary\", payload={\"user\": current_user.username})\n d = DictionaryManager.create_dictionary(\n db,\n name=payload.name,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n created_by=current_user.username,\n description=payload.description,\n is_active=payload.is_active,\n )\n log.reflect(\"Dictionary created\", payload={\"id\": d.id})\n return _dict_to_response(d, 0)\n# #endregion create_dictionary\n" }, { "contract_id": "update_dictionary", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 119, - "end_line": 152, + "start_line": 122, + "end_line": 155, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24415,14 +24415,14 @@ } ], "anchor_syntax": "region", - "body": "# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Update an existing terminology dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Dictionary updated and returned with current entry count.\n# @SIDE_EFFECT Updates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}\")\nasync def update_dictionary(\n dictionary_id: str,\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update a terminology dictionary.\"\"\"\n with belief_scope(\"update_dictionary\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n d = DictionaryManager.update_dictionary(\n db,\n dict_id=dictionary_id,\n name=payload.name,\n description=payload.description,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n is_active=payload.is_active,\n )\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n logger.reflect(\"Dictionary updated\", {\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_dictionary\n" + "body": "# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Update an existing terminology dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Dictionary updated and returned with current entry count.\n# @SIDE_EFFECT Updates TerminologyDictionary record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}\")\nasync def update_dictionary(\n dictionary_id: str,\n payload: DictionaryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update a terminology dictionary.\"\"\"\n with belief_scope(\"update_dictionary\"):\n log.reason(\"Update dictionary\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n d = DictionaryManager.update_dictionary(\n db,\n dict_id=dictionary_id,\n name=payload.name,\n description=payload.description,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n is_active=payload.is_active,\n )\n from ....models.translate import DictionaryEntry\n entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count()\n log.reflect(\"Dictionary updated\", payload={\"id\": dictionary_id})\n return _dict_to_response(d, entry_count)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_dictionary\n" }, { "contract_id": "delete_dictionary", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 155, - "end_line": 179, + "start_line": 158, + "end_line": 182, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24458,14 +24458,14 @@ } ], "anchor_syntax": "region", - "body": "# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.\n# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use.\n# @POST Dictionary is deleted or 409 if attached to active jobs.\n# @SIDE_EFFECT Deletes TerminologyDictionary record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"DELETE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a terminology dictionary.\"\"\"\n with belief_scope(\"delete_dictionary\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n DictionaryManager.delete_dictionary(db, dictionary_id)\n logger.reflect(\"Dictionary deleted\", {\"id\": dictionary_id})\n return None\n except ValueError as e:\n if \"active/scheduled\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary\n" + "body": "# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries]\n# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.\n# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use.\n# @POST Dictionary is deleted or 409 if attached to active jobs.\n# @SIDE_EFFECT Deletes TerminologyDictionary record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary(\n dictionary_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"DELETE\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a terminology dictionary.\"\"\"\n with belief_scope(\"delete_dictionary\"):\n log.reason(\"Delete dictionary\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n DictionaryManager.delete_dictionary(db, dictionary_id)\n log.reflect(\"Dictionary deleted\", payload={\"id\": dictionary_id})\n return None\n except ValueError as e:\n if \"active/scheduled\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary\n" }, { "contract_id": "list_dictionary_entries", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 186, - "end_line": 223, + "start_line": 189, + "end_line": 226, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24501,14 +24501,14 @@ } ], "anchor_syntax": "region", - "body": "# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF List entries for a dictionary with pagination.\n# @PRE User has translate.dictionary.view permission. Dictionary exists.\n# @POST Returns paginated list of entries.\n# @SIDE_EFFECT Reads DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}/entries\")\nasync def list_dictionary_entries(\n dictionary_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List entries for a dictionary.\"\"\"\n with belief_scope(\"list_dictionary_entries\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)\n items = [\n {\n \"id\": e.id,\n \"dictionary_id\": e.dictionary_id,\n \"source_term\": e.source_term,\n \"source_term_normalized\": e.source_term_normalized,\n \"target_term\": e.target_term,\n \"context_notes\": e.context_notes,\n \"created_at\": e.created_at,\n \"updated_at\": e.updated_at,\n }\n for e in entries\n ]\n logger.reflect(\"Entries listed\", {\"dict_id\": dictionary_id, \"total\": total})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion list_dictionary_entries\n" + "body": "# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF List entries for a dictionary with pagination.\n# @PRE User has translate.dictionary.view permission. Dictionary exists.\n# @POST Returns paginated list of entries.\n# @SIDE_EFFECT Reads DictionaryEntry records from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.get(\"/dictionaries/{dictionary_id}/entries\")\nasync def list_dictionary_entries(\n dictionary_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List entries for a dictionary.\"\"\"\n with belief_scope(\"list_dictionary_entries\"):\n log.reason(\"List dictionary entries\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)\n items = [\n {\n \"id\": e.id,\n \"dictionary_id\": e.dictionary_id,\n \"source_term\": e.source_term,\n \"source_term_normalized\": e.source_term_normalized,\n \"target_term\": e.target_term,\n \"context_notes\": e.context_notes,\n \"created_at\": e.created_at,\n \"updated_at\": e.updated_at,\n }\n for e in entries\n ]\n log.reflect(\"Entries listed\", payload={\"dict_id\": dictionary_id, \"total\": total})\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion list_dictionary_entries\n" }, { "contract_id": "add_dictionary_entry", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 226, - "end_line": 265, + "start_line": 229, + "end_line": 268, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24544,14 +24544,14 @@ } ], "anchor_syntax": "region", - "body": "# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Add a new entry to a dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Entry created and returned.\n# @SIDE_EFFECT Creates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/entries\", status_code=status.HTTP_201_CREATED)\nasync def add_dictionary_entry(\n dictionary_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Add a new entry to a dictionary.\"\"\"\n with belief_scope(\"add_dictionary_entry\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n try:\n entry = DictionaryManager.add_entry(\n db, dictionary_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n logger.reflect(\"Entry added\", {\"entry_id\": entry.id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion add_dictionary_entry\n" + "body": "# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Add a new entry to a dictionary.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists.\n# @POST Entry created and returned.\n# @SIDE_EFFECT Creates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/entries\", status_code=status.HTTP_201_CREATED)\nasync def add_dictionary_entry(\n dictionary_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Add a new entry to a dictionary.\"\"\"\n with belief_scope(\"add_dictionary_entry\"):\n log.reason(\"Add dictionary entry\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n try:\n entry = DictionaryManager.add_entry(\n db, dictionary_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n log.reflect(\"Entry added\", payload={\"entry_id\": entry.id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion add_dictionary_entry\n" }, { "contract_id": "edit_dictionary_entry", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 268, - "end_line": 308, + "start_line": 271, + "end_line": 311, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24587,14 +24587,14 @@ } ], "anchor_syntax": "region", - "body": "# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Update an existing dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry updated and returned.\n# @SIDE_EFFECT Updates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}/entries/{entry_id}\")\nasync def edit_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update an existing dictionary entry.\"\"\"\n with belief_scope(\"edit_dictionary_entry\"):\n logger.reason(f\"Entry: {entry_id}, User: {current_user.username}\")\n try:\n entry = DictionaryManager.edit_entry(\n db, entry_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n logger.reflect(\"Entry updated\", {\"entry_id\": entry_id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion edit_dictionary_entry\n" + "body": "# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Update an existing dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry updated and returned.\n# @SIDE_EFFECT Updates DictionaryEntry record in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.put(\"/dictionaries/{dictionary_id}/entries/{entry_id}\")\nasync def edit_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n payload: DictionaryEntryCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Update an existing dictionary entry.\"\"\"\n with belief_scope(\"edit_dictionary_entry\"):\n log.reason(\"Edit dictionary entry\", payload={\"entry_id\": entry_id, \"user\": current_user.username})\n try:\n entry = DictionaryManager.edit_entry(\n db, entry_id,\n source_term=payload.source_term,\n target_term=payload.target_term,\n context_notes=payload.context_notes,\n )\n log.reflect(\"Entry updated\", payload={\"entry_id\": entry_id})\n return {\n \"id\": entry.id,\n \"dictionary_id\": entry.dictionary_id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"context_notes\": entry.context_notes,\n \"created_at\": entry.created_at,\n \"updated_at\": entry.updated_at,\n }\n except ValueError as e:\n if \"already exists\" in str(e):\n raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e))\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion edit_dictionary_entry\n" }, { "contract_id": "delete_dictionary_entry", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 311, - "end_line": 334, + "start_line": 314, + "end_line": 337, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24630,14 +24630,14 @@ } ], "anchor_syntax": "region", - "body": "# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Delete a dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry is deleted.\n# @SIDE_EFFECT Deletes DictionaryEntry record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}/entries/{entry_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a dictionary entry.\"\"\"\n with belief_scope(\"delete_dictionary_entry\"):\n logger.reason(f\"Entry: {entry_id}, User: {current_user.username}\")\n try:\n DictionaryManager.delete_entry(db, entry_id)\n logger.reflect(\"Entry deleted\", {\"entry_id\": entry_id})\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary_entry\n" + "body": "# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries]\n# @BRIEF Delete a dictionary entry.\n# @PRE User has translate.dictionary.edit permission. Entry exists.\n# @POST Entry is deleted.\n# @SIDE_EFFECT Deletes DictionaryEntry record from DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.delete(\"/dictionaries/{dictionary_id}/entries/{entry_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_dictionary_entry(\n dictionary_id: str,\n entry_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Delete a dictionary entry.\"\"\"\n with belief_scope(\"delete_dictionary_entry\"):\n log.reason(\"Delete dictionary entry\", payload={\"entry_id\": entry_id, \"user\": current_user.username})\n try:\n DictionaryManager.delete_entry(db, entry_id)\n log.reflect(\"Entry deleted\", payload={\"entry_id\": entry_id})\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_dictionary_entry\n" }, { "contract_id": "import_dictionary_entries", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_dictionary_routes.py", - "start_line": 337, - "end_line": 378, + "start_line": 340, + "end_line": 381, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24673,7 +24673,7 @@ } ], "anchor_syntax": "region", - "body": "# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import]\n# @BRIEF Import entries into a terminology dictionary from CSV/TSV content.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV.\n# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/import\")\nasync def import_dictionary_entries(\n dictionary_id: str,\n payload: DictionaryImport,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Import entries into a terminology dictionary from CSV/TSV content.\"\"\"\n with belief_scope(\"import_dictionary_entries\"):\n logger.reason(f\"Dict: {dictionary_id}, User: {current_user.username}\")\n\n if payload.on_conflict not in (\"overwrite\", \"keep_existing\", \"cancel\"):\n raise HTTPException(\n status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,\n detail=\"on_conflict must be 'overwrite', 'keep_existing', or 'cancel'\",\n )\n\n try:\n result = DictionaryManager.import_entries(\n db,\n dict_id=dictionary_id,\n content=payload.content,\n delimiter=payload.delimiter,\n on_conflict=payload.on_conflict,\n preview_only=payload.preview_only,\n )\n logger.reflect(\"Import completed\", {\n \"dict_id\": dictionary_id,\n \"total\": result[\"total\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion import_dictionary_entries\n" + "body": "# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import]\n# @BRIEF Import entries into a terminology dictionary from CSV/TSV content.\n# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV.\n# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned.\n# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB.\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n@router.post(\"/dictionaries/{dictionary_id}/import\")\nasync def import_dictionary_entries(\n dictionary_id: str,\n payload: DictionaryImport,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.dictionary\", \"EDIT\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Import entries into a terminology dictionary from CSV/TSV content.\"\"\"\n with belief_scope(\"import_dictionary_entries\"):\n log.reason(\"Import dictionary entries\", payload={\"dict_id\": dictionary_id, \"user\": current_user.username})\n\n if payload.on_conflict not in (\"overwrite\", \"keep_existing\", \"cancel\"):\n raise HTTPException(\n status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,\n detail=\"on_conflict must be 'overwrite', 'keep_existing', or 'cancel'\",\n )\n\n try:\n result = DictionaryManager.import_entries(\n db,\n dict_id=dictionary_id,\n content=payload.content,\n delimiter=payload.delimiter,\n on_conflict=payload.on_conflict,\n preview_only=payload.preview_only,\n )\n log.reflect(\"Import completed\", payload={\n \"dict_id\": dictionary_id,\n \"total\": result[\"total\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion import_dictionary_entries\n" }, { "contract_id": "TranslateHelpersModule", @@ -24813,7 +24813,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_job_routes.py", "start_line": 1, - "end_line": 268, + "end_line": 272, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24874,14 +24874,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateJobRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,jobs]\n# @BRIEF Translation Job CRUD and datasource column routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate.job permissions.\n# @POST Translation job CRUD completed or datasource columns fetched.\n# @SIDE_EFFECT Reads/writes TranslationJob records to DB; fetches datasource metadata from Superset API.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.service import (\n TranslateJobService,\n job_to_response,\n get_datasource_columns as fetch_datasource_columns_service,\n)\nfrom ....schemas.translate import (\n TranslateJobCreate,\n TranslateJobUpdate,\n TranslateJobResponse,\n DatasourceColumnsResponse,\n DuplicateJobResponse,\n)\n\nfrom ._router import router\n\n\n# ============================================================\n# Translation Job CRUD\n# ============================================================\n\n# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF List all translation jobs with pagination and optional status filter.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs\", response_model=List[TranslateJobResponse])\nasync def list_jobs(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all translation jobs.\"\"\"\n logger.info(f\"[translate_routes][list_jobs] User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n total, jobs = service.list_jobs(\n page=page,\n page_size=page_size,\n status_filter=status,\n )\n results = []\n for job in jobs:\n dict_ids = service.get_job_dictionary_ids(job.id)\n results.append(job_to_response(job, dict_ids))\n return results\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_jobs\n\n\n# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Get a single translation job by ID.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def get_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get a translation job by ID.\"\"\"\n logger.info(f\"[translate_routes][get_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.get_job(job_id)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job\n\n\n# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Create a new translation job.\n# @PRE User has translate.job.create permission. Payload validated.\n# @POST Translation job created and returned.\n# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs\", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def create_job(\n payload: TranslateJobCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Create a new translation job.\"\"\"\n logger.info(f\"[translate_routes][create_job] User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.create_job(payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))\n# #endregion create_job\n\n\n# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Update an existing translation job.\n# @PRE User has translate.job.edit permission. Job exists.\n# @POST Translation job updated and returned.\n# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.put(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def update_job(\n job_id: str,\n payload: TranslateJobUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EDIT\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Update a translation job.\"\"\"\n logger.info(f\"[translate_routes][update_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.update_job(job_id, payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_job\n\n\n# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Delete a translation job.\n# @PRE User has translate.job.delete permission. Job exists.\n# @POST Translation job is deleted from DB.\n# @SIDE_EFFECT Deletes TranslationJob and associated records from DB.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.delete(\"/jobs/{job_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"DELETE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete a translation job.\"\"\"\n logger.info(f\"[translate_routes][delete_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n service.delete_job(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_job\n\n\n# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Duplicate a translation job.\n# @PRE User has translate.job.create permission. Source job exists.\n# @POST Duplicated job created with status DRAFT and returned.\n# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs/{job_id}/duplicate\", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def duplicate_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Duplicate a translation job.\"\"\"\n logger.info(f\"[translate_routes][duplicate_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n new_job = service.duplicate_job(job_id)\n return DuplicateJobResponse(\n id=new_job.id,\n name=new_job.name,\n message=\"Job duplicated successfully\",\n )\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion duplicate_job\n\n\n# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF List all Superset datasets available for translation jobs.\n# @PRE User has translate.job.view permission.\n# @POST Returns list of datasets with metadata.\n# @SIDE_EFFECT Queries Superset API for available datasets.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.get(\"/datasources\")\nasync def list_datasources(\n env_id: str = Query(..., description=\"Superset environment ID\"),\n search: Optional[str] = Query(None, description=\"Search by table name\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all Superset datasets available for translation jobs.\"\"\"\n with belief_scope(\"list_datasources\"):\n try:\n service = TranslateJobService(db, config_manager)\n datasets = service.fetch_available_datasources(env_id, search)\n return datasets\n except Exception as e:\n logger.error(f\"[translate_routes][list_datasources] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))\n# #endregion list_datasources\n\n\n# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF Get column metadata and database dialect for a Superset datasource.\n# @PRE User has translate.job.view permission. Datasource exists in Superset.\n# @POST Returns column list with metadata and database_dialect.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n@router.get(\"/datasources/{datasource_id}/columns\", response_model=DatasourceColumnsResponse)\nasync def get_datasource_columns(\n datasource_id: int,\n env_id: str = Query(..., description=\"Superset environment ID\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get column metadata and database dialect for a Superset datasource.\"\"\"\n logger.info(\n f\"[translate_routes][get_datasource_columns] \"\n f\"Datasource: {datasource_id}, Env: {env_id}, User: {current_user.username}\"\n )\n try:\n result = fetch_datasource_columns_service(\n datasource_id=datasource_id,\n env_id=env_id,\n config_manager=config_manager,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][get_datasource_columns] Error: {e}\")\n raise HTTPException(\n status_code=status.HTTP_502_BAD_GATEWAY,\n detail=f\"Failed to fetch datasource columns from Superset: {e}\",\n )\n# #endregion get_datasource_columns\n\n# #endregion TranslateJobRoutesModule\n" + "body": "# #region TranslateJobRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,jobs]\n# @BRIEF Translation Job CRUD and datasource column routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate.job permissions.\n# @POST Translation job CRUD completed or datasource columns fetched.\n# @SIDE_EFFECT Reads/writes TranslationJob records to DB; fetches datasource metadata from Superset API.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import belief_scope\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\n\nlog = MarkerLogger(\"TranslateJobRoutes\")\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.service import (\n TranslateJobService,\n job_to_response,\n get_datasource_columns as fetch_datasource_columns_service,\n)\nfrom ....schemas.translate import (\n TranslateJobCreate,\n TranslateJobUpdate,\n TranslateJobResponse,\n DatasourceColumnsResponse,\n DuplicateJobResponse,\n)\n\nfrom ._router import router\n\n\n# ============================================================\n# Translation Job CRUD\n# ============================================================\n\n# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF List all translation jobs with pagination and optional status filter.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs\", response_model=List[TranslateJobResponse])\nasync def list_jobs(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all translation jobs.\"\"\"\n log.reason(\"Listing jobs\", payload={\"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n total, jobs = service.list_jobs(\n page=page,\n page_size=page_size,\n status_filter=status,\n )\n results = []\n for job in jobs:\n dict_ids = service.get_job_dictionary_ids(job.id)\n results.append(job_to_response(job, dict_ids))\n return results\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_jobs\n\n\n# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Get a single translation job by ID.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def get_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get a translation job by ID.\"\"\"\n log.reason(\"Getting job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.get_job(job_id)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job\n\n\n# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Create a new translation job.\n# @PRE User has translate.job.create permission. Payload validated.\n# @POST Translation job created and returned.\n# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs\", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def create_job(\n payload: TranslateJobCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Create a new translation job.\"\"\"\n log.reason(\"Creating job\", payload={\"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.create_job(payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))\n# #endregion create_job\n\n\n# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Update an existing translation job.\n# @PRE User has translate.job.edit permission. Job exists.\n# @POST Translation job updated and returned.\n# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.put(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def update_job(\n job_id: str,\n payload: TranslateJobUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EDIT\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Update a translation job.\"\"\"\n log.reason(\"Updating job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.update_job(job_id, payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_job\n\n\n# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Delete a translation job.\n# @PRE User has translate.job.delete permission. Job exists.\n# @POST Translation job is deleted from DB.\n# @SIDE_EFFECT Deletes TranslationJob and associated records from DB.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.delete(\"/jobs/{job_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"DELETE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete a translation job.\"\"\"\n log.reason(\"Deleting job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n service.delete_job(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_job\n\n\n# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Duplicate a translation job.\n# @PRE User has translate.job.create permission. Source job exists.\n# @POST Duplicated job created with status DRAFT and returned.\n# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs/{job_id}/duplicate\", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def duplicate_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Duplicate a translation job.\"\"\"\n log.reason(\"Duplicating job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n new_job = service.duplicate_job(job_id)\n return DuplicateJobResponse(\n id=new_job.id,\n name=new_job.name,\n message=\"Job duplicated successfully\",\n )\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion duplicate_job\n\n\n# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF List all Superset datasets available for translation jobs.\n# @PRE User has translate.job.view permission.\n# @POST Returns list of datasets with metadata.\n# @SIDE_EFFECT Queries Superset API for available datasets.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.get(\"/datasources\")\nasync def list_datasources(\n env_id: str = Query(..., description=\"Superset environment ID\"),\n search: Optional[str] = Query(None, description=\"Search by table name\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all Superset datasets available for translation jobs.\"\"\"\n with belief_scope(\"list_datasources\"):\n try:\n service = TranslateJobService(db, config_manager)\n datasets = service.fetch_available_datasources(env_id, search)\n return datasets\n except Exception as e:\n log.explore(\"List datasources failed\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))\n# #endregion list_datasources\n\n\n# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF Get column metadata and database dialect for a Superset datasource.\n# @PRE User has translate.job.view permission. Datasource exists in Superset.\n# @POST Returns column list with metadata and database_dialect.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n@router.get(\"/datasources/{datasource_id}/columns\", response_model=DatasourceColumnsResponse)\nasync def get_datasource_columns(\n datasource_id: int,\n env_id: str = Query(..., description=\"Superset environment ID\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get column metadata and database dialect for a Superset datasource.\"\"\"\n log.reason(\"Getting datasource columns\", payload={\n \"datasource_id\": datasource_id,\n \"env_id\": env_id,\n \"user\": current_user.username,\n })\n try:\n result = fetch_datasource_columns_service(\n datasource_id=datasource_id,\n env_id=env_id,\n config_manager=config_manager,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Get datasource columns failed\", error=str(e))\n raise HTTPException(\n status_code=status.HTTP_502_BAD_GATEWAY,\n detail=f\"Failed to fetch datasource columns from Superset: {e}\",\n )\n# #endregion get_datasource_columns\n\n# #endregion TranslateJobRoutesModule\n" }, { "contract_id": "list_jobs", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 42, - "end_line": 71, + "start_line": 45, + "end_line": 74, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -24914,14 +24914,14 @@ } ], "anchor_syntax": "region", - "body": "# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF List all translation jobs with pagination and optional status filter.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs\", response_model=List[TranslateJobResponse])\nasync def list_jobs(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all translation jobs.\"\"\"\n logger.info(f\"[translate_routes][list_jobs] User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n total, jobs = service.list_jobs(\n page=page,\n page_size=page_size,\n status_filter=status,\n )\n results = []\n for job in jobs:\n dict_ids = service.get_job_dictionary_ids(job.id)\n results.append(job_to_response(job, dict_ids))\n return results\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_jobs\n" + "body": "# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF List all translation jobs with pagination and optional status filter.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs\", response_model=List[TranslateJobResponse])\nasync def list_jobs(\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all translation jobs.\"\"\"\n log.reason(\"Listing jobs\", payload={\"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n total, jobs = service.list_jobs(\n page=page,\n page_size=page_size,\n status_filter=status,\n )\n results = []\n for job in jobs:\n dict_ids = service.get_job_dictionary_ids(job.id)\n results.append(job_to_response(job, dict_ids))\n return results\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_jobs\n" }, { "contract_id": "get_job", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 74, - "end_line": 94, + "start_line": 77, + "end_line": 97, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -24954,14 +24954,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Get a single translation job by ID.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def get_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get a translation job by ID.\"\"\"\n logger.info(f\"[translate_routes][get_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.get_job(job_id)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job\n" + "body": "# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Get a single translation job by ID.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.get(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def get_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get a translation job by ID.\"\"\"\n log.reason(\"Getting job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.get_job(job_id)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job\n" }, { "contract_id": "create_job", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 97, - "end_line": 120, + "start_line": 100, + "end_line": 123, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -24997,14 +24997,14 @@ } ], "anchor_syntax": "region", - "body": "# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Create a new translation job.\n# @PRE User has translate.job.create permission. Payload validated.\n# @POST Translation job created and returned.\n# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs\", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def create_job(\n payload: TranslateJobCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Create a new translation job.\"\"\"\n logger.info(f\"[translate_routes][create_job] User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.create_job(payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))\n# #endregion create_job\n" + "body": "# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Create a new translation job.\n# @PRE User has translate.job.create permission. Payload validated.\n# @POST Translation job created and returned.\n# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs\", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def create_job(\n payload: TranslateJobCreate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Create a new translation job.\"\"\"\n log.reason(\"Creating job\", payload={\"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.create_job(payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e))\n# #endregion create_job\n" }, { "contract_id": "update_job", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 123, - "end_line": 147, + "start_line": 126, + "end_line": 150, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25040,14 +25040,14 @@ } ], "anchor_syntax": "region", - "body": "# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Update an existing translation job.\n# @PRE User has translate.job.edit permission. Job exists.\n# @POST Translation job updated and returned.\n# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.put(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def update_job(\n job_id: str,\n payload: TranslateJobUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EDIT\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Update a translation job.\"\"\"\n logger.info(f\"[translate_routes][update_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.update_job(job_id, payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_job\n" + "body": "# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Update an existing translation job.\n# @PRE User has translate.job.edit permission. Job exists.\n# @POST Translation job updated and returned.\n# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.put(\"/jobs/{job_id}\", response_model=TranslateJobResponse)\nasync def update_job(\n job_id: str,\n payload: TranslateJobUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EDIT\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Update a translation job.\"\"\"\n log.reason(\"Updating job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n job = service.update_job(job_id, payload)\n dict_ids = service.get_job_dictionary_ids(job.id)\n return job_to_response(job, dict_ids)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion update_job\n" }, { "contract_id": "delete_job", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 150, - "end_line": 171, + "start_line": 153, + "end_line": 174, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25083,14 +25083,14 @@ } ], "anchor_syntax": "region", - "body": "# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Delete a translation job.\n# @PRE User has translate.job.delete permission. Job exists.\n# @POST Translation job is deleted from DB.\n# @SIDE_EFFECT Deletes TranslationJob and associated records from DB.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.delete(\"/jobs/{job_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"DELETE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete a translation job.\"\"\"\n logger.info(f\"[translate_routes][delete_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n service.delete_job(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_job\n" + "body": "# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Delete a translation job.\n# @PRE User has translate.job.delete permission. Job exists.\n# @POST Translation job is deleted from DB.\n# @SIDE_EFFECT Deletes TranslationJob and associated records from DB.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.delete(\"/jobs/{job_id}\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"DELETE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete a translation job.\"\"\"\n log.reason(\"Deleting job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n service.delete_job(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_job\n" }, { "contract_id": "duplicate_job", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 174, - "end_line": 200, + "start_line": 177, + "end_line": 203, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25126,14 +25126,14 @@ } ], "anchor_syntax": "region", - "body": "# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Duplicate a translation job.\n# @PRE User has translate.job.create permission. Source job exists.\n# @POST Duplicated job created with status DRAFT and returned.\n# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs/{job_id}/duplicate\", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def duplicate_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Duplicate a translation job.\"\"\"\n logger.info(f\"[translate_routes][duplicate_job] Job: {job_id}, User: {current_user.username}\")\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n new_job = service.duplicate_job(job_id)\n return DuplicateJobResponse(\n id=new_job.id,\n name=new_job.name,\n message=\"Job duplicated successfully\",\n )\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion duplicate_job\n" + "body": "# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs]\n# @BRIEF Duplicate a translation job.\n# @PRE User has translate.job.create permission. Source job exists.\n# @POST Duplicated job created with status DRAFT and returned.\n# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n@router.post(\"/jobs/{job_id}/duplicate\", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED)\nasync def duplicate_job(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"CREATE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Duplicate a translation job.\"\"\"\n log.reason(\"Duplicating job\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n service = TranslateJobService(db, config_manager, current_user.username)\n new_job = service.duplicate_job(job_id)\n return DuplicateJobResponse(\n id=new_job.id,\n name=new_job.name,\n message=\"Job duplicated successfully\",\n )\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion duplicate_job\n" }, { "contract_id": "list_datasources", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 203, - "end_line": 228, + "start_line": 206, + "end_line": 231, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25175,14 +25175,14 @@ } ], "anchor_syntax": "region", - "body": "# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF List all Superset datasets available for translation jobs.\n# @PRE User has translate.job.view permission.\n# @POST Returns list of datasets with metadata.\n# @SIDE_EFFECT Queries Superset API for available datasets.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.get(\"/datasources\")\nasync def list_datasources(\n env_id: str = Query(..., description=\"Superset environment ID\"),\n search: Optional[str] = Query(None, description=\"Search by table name\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all Superset datasets available for translation jobs.\"\"\"\n with belief_scope(\"list_datasources\"):\n try:\n service = TranslateJobService(db, config_manager)\n datasets = service.fetch_available_datasources(env_id, search)\n return datasets\n except Exception as e:\n logger.error(f\"[translate_routes][list_datasources] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))\n# #endregion list_datasources\n" + "body": "# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF List all Superset datasets available for translation jobs.\n# @PRE User has translate.job.view permission.\n# @POST Returns list of datasets with metadata.\n# @SIDE_EFFECT Queries Superset API for available datasets.\n# @RELATION DEPENDS_ON -> [TranslateJobService]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.get(\"/datasources\")\nasync def list_datasources(\n env_id: str = Query(..., description=\"Superset environment ID\"),\n search: Optional[str] = Query(None, description=\"Search by table name\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"List all Superset datasets available for translation jobs.\"\"\"\n with belief_scope(\"list_datasources\"):\n try:\n service = TranslateJobService(db, config_manager)\n datasets = service.fetch_available_datasources(env_id, search)\n return datasets\n except Exception as e:\n log.explore(\"List datasources failed\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e))\n# #endregion list_datasources\n" }, { "contract_id": "get_datasource_columns", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_job_routes.py", - "start_line": 231, - "end_line": 266, + "start_line": 234, + "end_line": 270, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25224,14 +25224,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF Get column metadata and database dialect for a Superset datasource.\n# @PRE User has translate.job.view permission. Datasource exists in Superset.\n# @POST Returns column list with metadata and database_dialect.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n@router.get(\"/datasources/{datasource_id}/columns\", response_model=DatasourceColumnsResponse)\nasync def get_datasource_columns(\n datasource_id: int,\n env_id: str = Query(..., description=\"Superset environment ID\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get column metadata and database dialect for a Superset datasource.\"\"\"\n logger.info(\n f\"[translate_routes][get_datasource_columns] \"\n f\"Datasource: {datasource_id}, Env: {env_id}, User: {current_user.username}\"\n )\n try:\n result = fetch_datasource_columns_service(\n datasource_id=datasource_id,\n env_id=env_id,\n config_manager=config_manager,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][get_datasource_columns] Error: {e}\")\n raise HTTPException(\n status_code=status.HTTP_502_BAD_GATEWAY,\n detail=f\"Failed to fetch datasource columns from Superset: {e}\",\n )\n# #endregion get_datasource_columns\n" + "body": "# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources]\n# @BRIEF Get column metadata and database dialect for a Superset datasource.\n# @PRE User has translate.job.view permission. Datasource exists in Superset.\n# @POST Returns column list with metadata and database_dialect.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n@router.get(\"/datasources/{datasource_id}/columns\", response_model=DatasourceColumnsResponse)\nasync def get_datasource_columns(\n datasource_id: int,\n env_id: str = Query(..., description=\"Superset environment ID\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get column metadata and database dialect for a Superset datasource.\"\"\"\n log.reason(\"Getting datasource columns\", payload={\n \"datasource_id\": datasource_id,\n \"env_id\": env_id,\n \"user\": current_user.username,\n })\n try:\n result = fetch_datasource_columns_service(\n datasource_id=datasource_id,\n env_id=env_id,\n config_manager=config_manager,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Get datasource columns failed\", error=str(e))\n raise HTTPException(\n status_code=status.HTTP_502_BAD_GATEWAY,\n detail=f\"Failed to fetch datasource columns from Superset: {e}\",\n )\n# #endregion get_datasource_columns\n" }, { "contract_id": "TranslateMetricsRoutesModule", "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_metrics_routes.py", "start_line": 1, - "end_line": 66, + "end_line": 67, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25277,14 +25277,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateMetricsRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,metrics]\n# @BRIEF Translation Metrics endpoints providing aggregated stats on runs and corrections.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission\nfrom ....plugins.translate.metrics import TranslationMetrics\n\nfrom ._router import router\n\n\n# ============================================================\n# Metrics\n# ============================================================\n\n# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get translation metrics across all jobs, optionally filtered by job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/metrics\")\nasync def get_metrics(\n job_id: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get translation metrics.\"\"\"\n logger.info(f\"[translate_routes][get_metrics] User: {current_user.username}\")\n try:\n metrics_svc = TranslationMetrics(db)\n if job_id:\n return metrics_svc.get_job_metrics(job_id)\n return metrics_svc.get_all_metrics()\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_metrics\n\n\n# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get aggregated metrics for a specific translation job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/jobs/{job_id}/metrics\")\nasync def get_job_metrics(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get aggregated metrics for a specific job.\"\"\"\n logger.info(f\"[translate_routes][get_job_metrics] Job: {job_id}, User: {current_user.username}\")\n try:\n metrics_svc = TranslationMetrics(db)\n return metrics_svc.get_job_metrics(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job_metrics\n\n# #endregion TranslateMetricsRoutesModule\n" + "body": "# #region TranslateMetricsRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,metrics]\n# @BRIEF Translation Metrics endpoints providing aggregated stats on runs and corrections.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission\nfrom ....plugins.translate.metrics import TranslationMetrics\n\nfrom ._router import router\n\nlog = MarkerLogger(\"TranslateMetricsRoutes\")\n\n# ============================================================\n# Metrics\n# ============================================================\n\n# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get translation metrics across all jobs, optionally filtered by job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/metrics\")\nasync def get_metrics(\n job_id: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get translation metrics.\"\"\"\n log.reason(\"Metrics request\", payload={\"user\": current_user.username})\n try:\n metrics_svc = TranslationMetrics(db)\n if job_id:\n return metrics_svc.get_job_metrics(job_id)\n return metrics_svc.get_all_metrics()\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_metrics\n\n\n# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get aggregated metrics for a specific translation job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/jobs/{job_id}/metrics\")\nasync def get_job_metrics(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get aggregated metrics for a specific job.\"\"\"\n log.reason(\"Job metrics request\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n metrics_svc = TranslationMetrics(db)\n return metrics_svc.get_job_metrics(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job_metrics\n\n# #endregion TranslateMetricsRoutesModule\n" }, { "contract_id": "get_metrics", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_metrics_routes.py", - "start_line": 25, - "end_line": 44, + "start_line": 26, + "end_line": 45, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25317,14 +25317,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get translation metrics across all jobs, optionally filtered by job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/metrics\")\nasync def get_metrics(\n job_id: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get translation metrics.\"\"\"\n logger.info(f\"[translate_routes][get_metrics] User: {current_user.username}\")\n try:\n metrics_svc = TranslationMetrics(db)\n if job_id:\n return metrics_svc.get_job_metrics(job_id)\n return metrics_svc.get_all_metrics()\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_metrics\n" + "body": "# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get translation metrics across all jobs, optionally filtered by job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/metrics\")\nasync def get_metrics(\n job_id: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get translation metrics.\"\"\"\n log.reason(\"Metrics request\", payload={\"user\": current_user.username})\n try:\n metrics_svc = TranslationMetrics(db)\n if job_id:\n return metrics_svc.get_job_metrics(job_id)\n return metrics_svc.get_all_metrics()\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_metrics\n" }, { "contract_id": "get_job_metrics", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_metrics_routes.py", - "start_line": 47, - "end_line": 64, + "start_line": 48, + "end_line": 65, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25357,14 +25357,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get aggregated metrics for a specific translation job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/jobs/{job_id}/metrics\")\nasync def get_job_metrics(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get aggregated metrics for a specific job.\"\"\"\n logger.info(f\"[translate_routes][get_job_metrics] Job: {job_id}, User: {current_user.username}\")\n try:\n metrics_svc = TranslationMetrics(db)\n return metrics_svc.get_job_metrics(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job_metrics\n" + "body": "# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics]\n# @BRIEF Get aggregated metrics for a specific translation job.\n# @RELATION DEPENDS_ON -> [TranslationMetrics]\n@router.get(\"/jobs/{job_id}/metrics\")\nasync def get_job_metrics(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.metrics\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get aggregated metrics for a specific job.\"\"\"\n log.reason(\"Job metrics request\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n metrics_svc = TranslationMetrics(db)\n return metrics_svc.get_job_metrics(job_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_job_metrics\n" }, { "contract_id": "TranslatePreviewRoutesModule", "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_preview_routes.py", "start_line": 1, - "end_line": 208, + "end_line": 210, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25419,14 +25419,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslatePreviewRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,preview]\n# @BRIEF Translation Preview session management routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.\n# @POST Preview sessions created, rows reviewed/accepted, records queried.\n# @SIDE_EFFECT Creates/updates preview sessions and rows in DB; fetches sample data from Superset; calls LLM provider.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.preview import TranslationPreview\nfrom ....schemas.translate import (\n PreviewRequest,\n PreviewRowUpdate,\n PreviewAcceptResponse,\n PreviewRow,\n)\n\nfrom ._router import router\n\n\n# ============================================================\n# Preview\n# ============================================================\n\n# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Preview a translation before applying it.\n# @PRE User has translate.job.execute permission. Job exists.\n# @POST Preview session created with sample rows and cost estimation.\n# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.post(\"/jobs/{job_id}/preview\", status_code=status.HTTP_201_CREATED)\nasync def preview_translation(\n job_id: str,\n payload: PreviewRequest = None,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview a translation before applying it.\"\"\"\n logger.info(f\"[translate_routes][preview_translation] Job: {job_id}, User: {current_user.username}\")\n if payload is None:\n payload = PreviewRequest()\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.preview_rows(\n job_id=job_id,\n sample_size=payload.sample_size,\n prompt_template=payload.prompt_template,\n env_id=payload.env_id,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][preview_translation] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Preview failed: {e}\")\n# #endregion preview_translation\n\n\n# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Approve, edit, or reject a preview row.\n# @PRE User has translate.job.execute permission. Preview row exists.\n# @POST Preview row status updated to APPROVED/REJECTED/EDITED.\n# @SIDE_EFFECT Updates PreviewRecord status in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.put(\"/jobs/{job_id}/preview/rows/{row_key}\")\nasync def update_preview_row(\n job_id: str,\n row_key: str,\n payload: PreviewRowUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Approve, edit, or reject a preview row.\"\"\"\n logger.info(f\"[translate_routes][update_preview_row] Job: {job_id}, Row: {row_key}, Action: {payload.action}\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.update_preview_row(\n job_id=job_id,\n row_id=row_key,\n action=payload.action,\n translation=payload.translation,\n feedback=payload.feedback,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion update_preview_row\n\n\n# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Accept a preview session, marking it as the quality gate for full execution.\n# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session.\n# @POST Preview session status set to APPLIED. Full execution can proceed.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/jobs/{job_id}/preview/accept\")\nasync def accept_preview_session(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Accept a preview session, enabling full translation execution.\"\"\"\n logger.info(f\"[translate_routes][accept_preview_session] Job: {job_id}, User: {current_user.username}\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion accept_preview_session\n\n\n# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Apply a preview session (alias for accept when accepting at session level).\n# @PRE User has translate.job.execute permission. Session exists.\n# @POST Preview session applied and marked as quality gate.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/preview/{session_id}/apply\")\nasync def apply_preview(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Apply a preview session by session ID.\"\"\"\n logger.info(f\"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}\")\n # Find job_id from session\n from ....models.translate import TranslationPreviewSession\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=session.job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion apply_preview\n\n\n# ============================================================\n# Preview Records\n# ============================================================\n\n# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Get records for a preview session.\n# @RELATION DEPENDS_ON -> [TranslationPreviewSession]\n# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]\n@router.get(\"/preview/{session_id}/records\", response_model=List[PreviewRow])\nasync def get_preview_records(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get records for a preview session.\"\"\"\n logger.info(f\"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}\")\n try:\n from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n records = (\n db.query(TranslationPreviewRecord)\n .filter(TranslationPreviewRecord.session_id == session_id)\n .all()\n )\n return [\n PreviewRow(\n id=r.id,\n source_sql=r.source_sql,\n target_sql=r.target_sql,\n source_object_type=r.source_object_type,\n source_object_id=r.source_object_id,\n source_object_name=r.source_object_name,\n status=r.status,\n feedback=r.feedback,\n )\n for r in records\n ]\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"[translate_routes][get_preview_records] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_preview_records\n\n# #endregion TranslatePreviewRoutesModule\n" + "body": "# #region TranslatePreviewRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,preview]\n# @BRIEF Translation Preview session management routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.\n# @POST Preview sessions created, rows reviewed/accepted, records queried.\n# @SIDE_EFFECT Creates/updates preview sessions and rows in DB; fetches sample data from Superset; calls LLM provider.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\n\nlog = MarkerLogger(\"TranslatePreviewRoutes\")\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.preview import TranslationPreview\nfrom ....schemas.translate import (\n PreviewRequest,\n PreviewRowUpdate,\n PreviewAcceptResponse,\n PreviewRow,\n)\n\nfrom ._router import router\n\n\n# ============================================================\n# Preview\n# ============================================================\n\n# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Preview a translation before applying it.\n# @PRE User has translate.job.execute permission. Job exists.\n# @POST Preview session created with sample rows and cost estimation.\n# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.post(\"/jobs/{job_id}/preview\", status_code=status.HTTP_201_CREATED)\nasync def preview_translation(\n job_id: str,\n payload: PreviewRequest = None,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview a translation before applying it.\"\"\"\n log.reason(\"Preview translation\", payload={\"job_id\": job_id, \"user\": current_user.username})\n if payload is None:\n payload = PreviewRequest()\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.preview_rows(\n job_id=job_id,\n sample_size=payload.sample_size,\n prompt_template=payload.prompt_template,\n env_id=payload.env_id,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Preview translation error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Preview failed: {e}\")\n# #endregion preview_translation\n\n\n# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Approve, edit, or reject a preview row.\n# @PRE User has translate.job.execute permission. Preview row exists.\n# @POST Preview row status updated to APPROVED/REJECTED/EDITED.\n# @SIDE_EFFECT Updates PreviewRecord status in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.put(\"/jobs/{job_id}/preview/rows/{row_key}\")\nasync def update_preview_row(\n job_id: str,\n row_key: str,\n payload: PreviewRowUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Approve, edit, or reject a preview row.\"\"\"\n log.reason(\"Update preview row\", payload={\"job_id\": job_id, \"row\": row_key, \"action\": payload.action})\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.update_preview_row(\n job_id=job_id,\n row_id=row_key,\n action=payload.action,\n translation=payload.translation,\n feedback=payload.feedback,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion update_preview_row\n\n\n# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Accept a preview session, marking it as the quality gate for full execution.\n# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session.\n# @POST Preview session status set to APPLIED. Full execution can proceed.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/jobs/{job_id}/preview/accept\")\nasync def accept_preview_session(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Accept a preview session, enabling full translation execution.\"\"\"\n log.reason(\"Accept preview session\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion accept_preview_session\n\n\n# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Apply a preview session (alias for accept when accepting at session level).\n# @PRE User has translate.job.execute permission. Session exists.\n# @POST Preview session applied and marked as quality gate.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/preview/{session_id}/apply\")\nasync def apply_preview(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Apply a preview session by session ID.\"\"\"\n log.reason(\"Apply preview\", payload={\"session_id\": session_id, \"user\": current_user.username})\n # Find job_id from session\n from ....models.translate import TranslationPreviewSession\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=session.job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion apply_preview\n\n\n# ============================================================\n# Preview Records\n# ============================================================\n\n# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Get records for a preview session.\n# @RELATION DEPENDS_ON -> [TranslationPreviewSession]\n# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]\n@router.get(\"/preview/{session_id}/records\", response_model=List[PreviewRow])\nasync def get_preview_records(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get records for a preview session.\"\"\"\n log.reason(\"Get preview records\", payload={\"session_id\": session_id, \"user\": current_user.username})\n try:\n from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n records = (\n db.query(TranslationPreviewRecord)\n .filter(TranslationPreviewRecord.session_id == session_id)\n .all()\n )\n return [\n PreviewRow(\n id=r.id,\n source_sql=r.source_sql,\n target_sql=r.target_sql,\n source_object_type=r.source_object_type,\n source_object_id=r.source_object_id,\n source_object_name=r.source_object_name,\n status=r.status,\n feedback=r.feedback,\n )\n for r in records\n ]\n except HTTPException:\n raise\n except Exception as e:\n log.explore(\"Get preview records error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_preview_records\n\n# #endregion TranslatePreviewRoutesModule\n" }, { "contract_id": "preview_translation", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_preview_routes.py", - "start_line": 36, - "end_line": 70, + "start_line": 38, + "end_line": 72, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25468,14 +25468,14 @@ } ], "anchor_syntax": "region", - "body": "# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Preview a translation before applying it.\n# @PRE User has translate.job.execute permission. Job exists.\n# @POST Preview session created with sample rows and cost estimation.\n# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.post(\"/jobs/{job_id}/preview\", status_code=status.HTTP_201_CREATED)\nasync def preview_translation(\n job_id: str,\n payload: PreviewRequest = None,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview a translation before applying it.\"\"\"\n logger.info(f\"[translate_routes][preview_translation] Job: {job_id}, User: {current_user.username}\")\n if payload is None:\n payload = PreviewRequest()\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.preview_rows(\n job_id=job_id,\n sample_size=payload.sample_size,\n prompt_template=payload.prompt_template,\n env_id=payload.env_id,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][preview_translation] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Preview failed: {e}\")\n# #endregion preview_translation\n" + "body": "# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Preview a translation before applying it.\n# @PRE User has translate.job.execute permission. Job exists.\n# @POST Preview session created with sample rows and cost estimation.\n# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n@router.post(\"/jobs/{job_id}/preview\", status_code=status.HTTP_201_CREATED)\nasync def preview_translation(\n job_id: str,\n payload: PreviewRequest = None,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview a translation before applying it.\"\"\"\n log.reason(\"Preview translation\", payload={\"job_id\": job_id, \"user\": current_user.username})\n if payload is None:\n payload = PreviewRequest()\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.preview_rows(\n job_id=job_id,\n sample_size=payload.sample_size,\n prompt_template=payload.prompt_template,\n env_id=payload.env_id,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Preview translation error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Preview failed: {e}\")\n# #endregion preview_translation\n" }, { "contract_id": "update_preview_row", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_preview_routes.py", - "start_line": 73, - "end_line": 103, + "start_line": 75, + "end_line": 105, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25511,14 +25511,14 @@ } ], "anchor_syntax": "region", - "body": "# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Approve, edit, or reject a preview row.\n# @PRE User has translate.job.execute permission. Preview row exists.\n# @POST Preview row status updated to APPROVED/REJECTED/EDITED.\n# @SIDE_EFFECT Updates PreviewRecord status in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.put(\"/jobs/{job_id}/preview/rows/{row_key}\")\nasync def update_preview_row(\n job_id: str,\n row_key: str,\n payload: PreviewRowUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Approve, edit, or reject a preview row.\"\"\"\n logger.info(f\"[translate_routes][update_preview_row] Job: {job_id}, Row: {row_key}, Action: {payload.action}\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.update_preview_row(\n job_id=job_id,\n row_id=row_key,\n action=payload.action,\n translation=payload.translation,\n feedback=payload.feedback,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion update_preview_row\n" + "body": "# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Approve, edit, or reject a preview row.\n# @PRE User has translate.job.execute permission. Preview row exists.\n# @POST Preview row status updated to APPROVED/REJECTED/EDITED.\n# @SIDE_EFFECT Updates PreviewRecord status in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.put(\"/jobs/{job_id}/preview/rows/{row_key}\")\nasync def update_preview_row(\n job_id: str,\n row_key: str,\n payload: PreviewRowUpdate,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Approve, edit, or reject a preview row.\"\"\"\n log.reason(\"Update preview row\", payload={\"job_id\": job_id, \"row\": row_key, \"action\": payload.action})\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.update_preview_row(\n job_id=job_id,\n row_id=row_key,\n action=payload.action,\n translation=payload.translation,\n feedback=payload.feedback,\n )\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion update_preview_row\n" }, { "contract_id": "accept_preview_session", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_preview_routes.py", - "start_line": 106, - "end_line": 128, + "start_line": 108, + "end_line": 130, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25554,14 +25554,14 @@ } ], "anchor_syntax": "region", - "body": "# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Accept a preview session, marking it as the quality gate for full execution.\n# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session.\n# @POST Preview session status set to APPLIED. Full execution can proceed.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/jobs/{job_id}/preview/accept\")\nasync def accept_preview_session(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Accept a preview session, enabling full translation execution.\"\"\"\n logger.info(f\"[translate_routes][accept_preview_session] Job: {job_id}, User: {current_user.username}\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion accept_preview_session\n" + "body": "# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Accept a preview session, marking it as the quality gate for full execution.\n# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session.\n# @POST Preview session status set to APPLIED. Full execution can proceed.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/jobs/{job_id}/preview/accept\")\nasync def accept_preview_session(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Accept a preview session, enabling full translation execution.\"\"\"\n log.reason(\"Accept preview session\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion accept_preview_session\n" }, { "contract_id": "apply_preview", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_preview_routes.py", - "start_line": 131, - "end_line": 158, + "start_line": 133, + "end_line": 160, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25597,14 +25597,14 @@ } ], "anchor_syntax": "region", - "body": "# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Apply a preview session (alias for accept when accepting at session level).\n# @PRE User has translate.job.execute permission. Session exists.\n# @POST Preview session applied and marked as quality gate.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/preview/{session_id}/apply\")\nasync def apply_preview(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Apply a preview session by session ID.\"\"\"\n logger.info(f\"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}\")\n # Find job_id from session\n from ....models.translate import TranslationPreviewSession\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=session.job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion apply_preview\n" + "body": "# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Apply a preview session (alias for accept when accepting at session level).\n# @PRE User has translate.job.execute permission. Session exists.\n# @POST Preview session applied and marked as quality gate.\n# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB.\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n@router.post(\"/preview/{session_id}/apply\")\nasync def apply_preview(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Apply a preview session by session ID.\"\"\"\n log.reason(\"Apply preview\", payload={\"session_id\": session_id, \"user\": current_user.username})\n # Find job_id from session\n from ....models.translate import TranslationPreviewSession\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n try:\n preview_service = TranslationPreview(db, config_manager, current_user.username)\n result = preview_service.accept_preview_session(job_id=session.job_id)\n return result\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion apply_preview\n" }, { "contract_id": "get_preview_records", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_preview_routes.py", - "start_line": 165, - "end_line": 206, + "start_line": 167, + "end_line": 208, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25643,7 +25643,7 @@ } ], "anchor_syntax": "region", - "body": "# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Get records for a preview session.\n# @RELATION DEPENDS_ON -> [TranslationPreviewSession]\n# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]\n@router.get(\"/preview/{session_id}/records\", response_model=List[PreviewRow])\nasync def get_preview_records(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get records for a preview session.\"\"\"\n logger.info(f\"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}\")\n try:\n from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n records = (\n db.query(TranslationPreviewRecord)\n .filter(TranslationPreviewRecord.session_id == session_id)\n .all()\n )\n return [\n PreviewRow(\n id=r.id,\n source_sql=r.source_sql,\n target_sql=r.target_sql,\n source_object_type=r.source_object_type,\n source_object_id=r.source_object_id,\n source_object_name=r.source_object_name,\n status=r.status,\n feedback=r.feedback,\n )\n for r in records\n ]\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"[translate_routes][get_preview_records] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_preview_records\n" + "body": "# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview]\n# @BRIEF Get records for a preview session.\n# @RELATION DEPENDS_ON -> [TranslationPreviewSession]\n# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]\n@router.get(\"/preview/{session_id}/records\", response_model=List[PreviewRow])\nasync def get_preview_records(\n session_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get records for a preview session.\"\"\"\n log.reason(\"Get preview records\", payload={\"session_id\": session_id, \"user\": current_user.username})\n try:\n from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord\n session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first()\n if not session:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f\"Preview session '{session_id}' not found\")\n records = (\n db.query(TranslationPreviewRecord)\n .filter(TranslationPreviewRecord.session_id == session_id)\n .all()\n )\n return [\n PreviewRow(\n id=r.id,\n source_sql=r.source_sql,\n target_sql=r.target_sql,\n source_object_type=r.source_object_type,\n source_object_id=r.source_object_id,\n source_object_name=r.source_object_name,\n status=r.status,\n feedback=r.feedback,\n )\n for r in records\n ]\n except HTTPException:\n raise\n except Exception as e:\n log.explore(\"Get preview records error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_preview_records\n" }, { "contract_id": "TranslateRouterModule", @@ -25718,7 +25718,7 @@ "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_run_list_routes.py", "start_line": 1, - "end_line": 207, + "end_line": 209, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25770,14 +25770,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,runs,list,detail]\n# @BRIEF Translation Run listing, detail, and CSV download routes (cross-job).\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.orchestrator import TranslationOrchestrator\nfrom ....plugins.translate.events import TranslationEventLog\n\nfrom ._router import router\n\n\n# ============================================================\n# List Runs (cross-job)\n# ============================================================\n\n# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list]\n# @BRIEF List all runs with cross-job filtering and pagination.\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs\")\nasync def list_runs(\n job_id: Optional[str] = Query(None),\n run_status: Optional[str] = Query(None, alias=\"status\"),\n trigger_type: Optional[str] = Query(None),\n created_by: Optional[str] = Query(None),\n date_from: Optional[str] = Query(None),\n date_to: Optional[str] = Query(None),\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all runs with cross-job filtering and pagination.\"\"\"\n logger.info(f\"[translate_routes][list_runs] User: {current_user.username}\")\n try:\n from ....models.translate import TranslationRun\n query = db.query(TranslationRun)\n\n if job_id:\n query = query.filter(TranslationRun.job_id == job_id)\n if run_status:\n query = query.filter(TranslationRun.status == run_status)\n if trigger_type:\n query = query.filter(TranslationRun.trigger_type == trigger_type)\n if created_by:\n query = query.filter(TranslationRun.created_by == created_by)\n if date_from:\n try:\n from datetime import datetime\n dt_from = datetime.fromisoformat(date_from)\n query = query.filter(TranslationRun.created_at >= dt_from)\n except ValueError:\n pass\n if date_to:\n try:\n from datetime import datetime\n dt_to = datetime.fromisoformat(date_to)\n query = query.filter(TranslationRun.created_at <= dt_to)\n except ValueError:\n pass\n\n total = query.count()\n runs = (\n query.order_by(TranslationRun.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n\n items = []\n for r in runs:\n items.append({\n \"id\": r.id,\n \"job_id\": r.job_id,\n \"status\": r.status,\n \"trigger_type\": r.trigger_type,\n \"started_at\": r.started_at.isoformat() if r.started_at else None,\n \"completed_at\": r.completed_at.isoformat() if r.completed_at else None,\n \"error_message\": r.error_message,\n \"total_records\": r.total_records or 0,\n \"successful_records\": r.successful_records or 0,\n \"failed_records\": r.failed_records or 0,\n \"skipped_records\": r.skipped_records or 0,\n \"insert_status\": r.insert_status,\n \"superset_execution_id\": r.superset_execution_id,\n \"config_hash\": r.config_hash,\n \"dict_snapshot_hash\": r.dict_snapshot_hash,\n \"created_by\": r.created_by,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n })\n\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except Exception as e:\n logger.error(f\"[translate_routes][list_runs] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_runs\n\n\n# ============================================================\n# Run Detail\n# ============================================================\n\n# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail]\n# @BRIEF Get detailed run info with config_snapshot, records, events.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs/{run_id}/detail\")\nasync def get_run_detail(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get detailed run info with config_snapshot, records, events.\"\"\"\n logger.info(f\"[translate_routes][get_run_detail] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n status_info = orch.get_run_status(run_id)\n records = orch.get_run_records(run_id, page=1, page_size=50)\n events = TranslationEventLog(db).query_events(run_id=run_id)\n event_summary = TranslationEventLog(db).get_run_event_summary(run_id)\n\n # Get config snapshot from run\n from ....models.translate import TranslationRun\n run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n config_snapshot = run.config_snapshot if run else None\n\n return {\n **status_info,\n \"config_snapshot\": config_snapshot,\n \"config_hash\": run.config_hash if run else None,\n \"records\": records,\n \"events\": events,\n \"event_invariants\": event_summary,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_detail\n\n\n# ============================================================\n# Download Skipped CSV\n# ============================================================\n\n# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv]\n# @BRIEF Download a CSV of skipped translation records from a run.\n# @RELATION DEPENDS_ON -> [TranslationRecord]\n@router.get(\"/runs/{run_id}/skipped.csv\")\nasync def download_skipped_csv(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Download skipped translation records as CSV.\"\"\"\n logger.info(f\"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}\")\n try:\n from ....models.translate import TranslationRecord\n from fastapi.responses import StreamingResponse\n import csv\n import io\n\n records = (\n db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run_id,\n TranslationRecord.status == \"SKIPPED\",\n )\n .all()\n )\n\n output = io.StringIO()\n writer = csv.writer(output)\n writer.writerow([\"id\", \"source_sql\", \"target_sql\", \"source_object_type\",\n \"source_object_id\", \"source_object_name\", \"error_message\"])\n for rec in records:\n writer.writerow([\n rec.id, rec.source_sql, rec.target_sql, rec.source_object_type,\n rec.source_object_id, rec.source_object_name, rec.error_message,\n ])\n\n output.seek(0)\n return StreamingResponse(\n iter([output.getvalue()]),\n media_type=\"text/csv\",\n headers={\"Content-Disposition\": f\"attachment; filename=skipped_{run_id}.csv\"},\n )\n except Exception as e:\n logger.error(f\"[translate_routes][download_skipped_csv] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion download_skipped_csv\n\n# #endregion TranslateRunListRoutesModule\n" + "body": "# #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,runs,list,detail]\n# @BRIEF Translation Run listing, detail, and CSV download routes (cross-job).\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\n\nlog = MarkerLogger(\"TranslateRunListRoutes\")\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.orchestrator import TranslationOrchestrator\nfrom ....plugins.translate.events import TranslationEventLog\n\nfrom ._router import router\n\n\n# ============================================================\n# List Runs (cross-job)\n# ============================================================\n\n# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list]\n# @BRIEF List all runs with cross-job filtering and pagination.\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs\")\nasync def list_runs(\n job_id: Optional[str] = Query(None),\n run_status: Optional[str] = Query(None, alias=\"status\"),\n trigger_type: Optional[str] = Query(None),\n created_by: Optional[str] = Query(None),\n date_from: Optional[str] = Query(None),\n date_to: Optional[str] = Query(None),\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all runs with cross-job filtering and pagination.\"\"\"\n log.reason(\"Listing runs\", payload={\"user\": current_user.username})\n try:\n from ....models.translate import TranslationRun\n query = db.query(TranslationRun)\n\n if job_id:\n query = query.filter(TranslationRun.job_id == job_id)\n if run_status:\n query = query.filter(TranslationRun.status == run_status)\n if trigger_type:\n query = query.filter(TranslationRun.trigger_type == trigger_type)\n if created_by:\n query = query.filter(TranslationRun.created_by == created_by)\n if date_from:\n try:\n from datetime import datetime\n dt_from = datetime.fromisoformat(date_from)\n query = query.filter(TranslationRun.created_at >= dt_from)\n except ValueError:\n pass\n if date_to:\n try:\n from datetime import datetime\n dt_to = datetime.fromisoformat(date_to)\n query = query.filter(TranslationRun.created_at <= dt_to)\n except ValueError:\n pass\n\n total = query.count()\n runs = (\n query.order_by(TranslationRun.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n\n items = []\n for r in runs:\n items.append({\n \"id\": r.id,\n \"job_id\": r.job_id,\n \"status\": r.status,\n \"trigger_type\": r.trigger_type,\n \"started_at\": r.started_at.isoformat() if r.started_at else None,\n \"completed_at\": r.completed_at.isoformat() if r.completed_at else None,\n \"error_message\": r.error_message,\n \"total_records\": r.total_records or 0,\n \"successful_records\": r.successful_records or 0,\n \"failed_records\": r.failed_records or 0,\n \"skipped_records\": r.skipped_records or 0,\n \"insert_status\": r.insert_status,\n \"superset_execution_id\": r.superset_execution_id,\n \"config_hash\": r.config_hash,\n \"dict_snapshot_hash\": r.dict_snapshot_hash,\n \"created_by\": r.created_by,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n })\n\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except Exception as e:\n log.explore(\"List runs error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_runs\n\n\n# ============================================================\n# Run Detail\n# ============================================================\n\n# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail]\n# @BRIEF Get detailed run info with config_snapshot, records, events.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs/{run_id}/detail\")\nasync def get_run_detail(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get detailed run info with config_snapshot, records, events.\"\"\"\n log.reason(\"Get run detail\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n status_info = orch.get_run_status(run_id)\n records = orch.get_run_records(run_id, page=1, page_size=50)\n events = TranslationEventLog(db).query_events(run_id=run_id)\n event_summary = TranslationEventLog(db).get_run_event_summary(run_id)\n\n # Get config snapshot from run\n from ....models.translate import TranslationRun\n run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n config_snapshot = run.config_snapshot if run else None\n\n return {\n **status_info,\n \"config_snapshot\": config_snapshot,\n \"config_hash\": run.config_hash if run else None,\n \"records\": records,\n \"events\": events,\n \"event_invariants\": event_summary,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_detail\n\n\n# ============================================================\n# Download Skipped CSV\n# ============================================================\n\n# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv]\n# @BRIEF Download a CSV of skipped translation records from a run.\n# @RELATION DEPENDS_ON -> [TranslationRecord]\n@router.get(\"/runs/{run_id}/skipped.csv\")\nasync def download_skipped_csv(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Download skipped translation records as CSV.\"\"\"\n log.reason(\"Download skipped CSV\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n from ....models.translate import TranslationRecord\n from fastapi.responses import StreamingResponse\n import csv\n import io\n\n records = (\n db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run_id,\n TranslationRecord.status == \"SKIPPED\",\n )\n .all()\n )\n\n output = io.StringIO()\n writer = csv.writer(output)\n writer.writerow([\"id\", \"source_sql\", \"target_sql\", \"source_object_type\",\n \"source_object_id\", \"source_object_name\", \"error_message\"])\n for rec in records:\n writer.writerow([\n rec.id, rec.source_sql, rec.target_sql, rec.source_object_type,\n rec.source_object_id, rec.source_object_name, rec.error_message,\n ])\n\n output.seek(0)\n return StreamingResponse(\n iter([output.getvalue()]),\n media_type=\"text/csv\",\n headers={\"Content-Disposition\": f\"attachment; filename=skipped_{run_id}.csv\"},\n )\n except Exception as e:\n log.explore(\"Download skipped CSV error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion download_skipped_csv\n\n# #endregion TranslateRunListRoutesModule\n" }, { "contract_id": "list_runs", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_list_routes.py", - "start_line": 28, - "end_line": 108, + "start_line": 30, + "end_line": 110, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25810,14 +25810,14 @@ } ], "anchor_syntax": "region", - "body": "# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list]\n# @BRIEF List all runs with cross-job filtering and pagination.\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs\")\nasync def list_runs(\n job_id: Optional[str] = Query(None),\n run_status: Optional[str] = Query(None, alias=\"status\"),\n trigger_type: Optional[str] = Query(None),\n created_by: Optional[str] = Query(None),\n date_from: Optional[str] = Query(None),\n date_to: Optional[str] = Query(None),\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all runs with cross-job filtering and pagination.\"\"\"\n logger.info(f\"[translate_routes][list_runs] User: {current_user.username}\")\n try:\n from ....models.translate import TranslationRun\n query = db.query(TranslationRun)\n\n if job_id:\n query = query.filter(TranslationRun.job_id == job_id)\n if run_status:\n query = query.filter(TranslationRun.status == run_status)\n if trigger_type:\n query = query.filter(TranslationRun.trigger_type == trigger_type)\n if created_by:\n query = query.filter(TranslationRun.created_by == created_by)\n if date_from:\n try:\n from datetime import datetime\n dt_from = datetime.fromisoformat(date_from)\n query = query.filter(TranslationRun.created_at >= dt_from)\n except ValueError:\n pass\n if date_to:\n try:\n from datetime import datetime\n dt_to = datetime.fromisoformat(date_to)\n query = query.filter(TranslationRun.created_at <= dt_to)\n except ValueError:\n pass\n\n total = query.count()\n runs = (\n query.order_by(TranslationRun.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n\n items = []\n for r in runs:\n items.append({\n \"id\": r.id,\n \"job_id\": r.job_id,\n \"status\": r.status,\n \"trigger_type\": r.trigger_type,\n \"started_at\": r.started_at.isoformat() if r.started_at else None,\n \"completed_at\": r.completed_at.isoformat() if r.completed_at else None,\n \"error_message\": r.error_message,\n \"total_records\": r.total_records or 0,\n \"successful_records\": r.successful_records or 0,\n \"failed_records\": r.failed_records or 0,\n \"skipped_records\": r.skipped_records or 0,\n \"insert_status\": r.insert_status,\n \"superset_execution_id\": r.superset_execution_id,\n \"config_hash\": r.config_hash,\n \"dict_snapshot_hash\": r.dict_snapshot_hash,\n \"created_by\": r.created_by,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n })\n\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except Exception as e:\n logger.error(f\"[translate_routes][list_runs] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_runs\n" + "body": "# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list]\n# @BRIEF List all runs with cross-job filtering and pagination.\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs\")\nasync def list_runs(\n job_id: Optional[str] = Query(None),\n run_status: Optional[str] = Query(None, alias=\"status\"),\n trigger_type: Optional[str] = Query(None),\n created_by: Optional[str] = Query(None),\n date_from: Optional[str] = Query(None),\n date_to: Optional[str] = Query(None),\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"List all runs with cross-job filtering and pagination.\"\"\"\n log.reason(\"Listing runs\", payload={\"user\": current_user.username})\n try:\n from ....models.translate import TranslationRun\n query = db.query(TranslationRun)\n\n if job_id:\n query = query.filter(TranslationRun.job_id == job_id)\n if run_status:\n query = query.filter(TranslationRun.status == run_status)\n if trigger_type:\n query = query.filter(TranslationRun.trigger_type == trigger_type)\n if created_by:\n query = query.filter(TranslationRun.created_by == created_by)\n if date_from:\n try:\n from datetime import datetime\n dt_from = datetime.fromisoformat(date_from)\n query = query.filter(TranslationRun.created_at >= dt_from)\n except ValueError:\n pass\n if date_to:\n try:\n from datetime import datetime\n dt_to = datetime.fromisoformat(date_to)\n query = query.filter(TranslationRun.created_at <= dt_to)\n except ValueError:\n pass\n\n total = query.count()\n runs = (\n query.order_by(TranslationRun.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n\n items = []\n for r in runs:\n items.append({\n \"id\": r.id,\n \"job_id\": r.job_id,\n \"status\": r.status,\n \"trigger_type\": r.trigger_type,\n \"started_at\": r.started_at.isoformat() if r.started_at else None,\n \"completed_at\": r.completed_at.isoformat() if r.completed_at else None,\n \"error_message\": r.error_message,\n \"total_records\": r.total_records or 0,\n \"successful_records\": r.successful_records or 0,\n \"failed_records\": r.failed_records or 0,\n \"skipped_records\": r.skipped_records or 0,\n \"insert_status\": r.insert_status,\n \"superset_execution_id\": r.superset_execution_id,\n \"config_hash\": r.config_hash,\n \"dict_snapshot_hash\": r.dict_snapshot_hash,\n \"created_by\": r.created_by,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n })\n\n return {\"items\": items, \"total\": total, \"page\": page, \"page_size\": page_size}\n except Exception as e:\n log.explore(\"List runs error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion list_runs\n" }, { "contract_id": "get_run_detail", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_list_routes.py", - "start_line": 115, - "end_line": 152, + "start_line": 117, + "end_line": 154, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25862,14 +25862,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail]\n# @BRIEF Get detailed run info with config_snapshot, records, events.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs/{run_id}/detail\")\nasync def get_run_detail(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get detailed run info with config_snapshot, records, events.\"\"\"\n logger.info(f\"[translate_routes][get_run_detail] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n status_info = orch.get_run_status(run_id)\n records = orch.get_run_records(run_id, page=1, page_size=50)\n events = TranslationEventLog(db).query_events(run_id=run_id)\n event_summary = TranslationEventLog(db).get_run_event_summary(run_id)\n\n # Get config snapshot from run\n from ....models.translate import TranslationRun\n run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n config_snapshot = run.config_snapshot if run else None\n\n return {\n **status_info,\n \"config_snapshot\": config_snapshot,\n \"config_hash\": run.config_hash if run else None,\n \"records\": records,\n \"events\": events,\n \"event_invariants\": event_summary,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_detail\n" + "body": "# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail]\n# @BRIEF Get detailed run info with config_snapshot, records, events.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n@router.get(\"/runs/{run_id}/detail\")\nasync def get_run_detail(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get detailed run info with config_snapshot, records, events.\"\"\"\n log.reason(\"Get run detail\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n status_info = orch.get_run_status(run_id)\n records = orch.get_run_records(run_id, page=1, page_size=50)\n events = TranslationEventLog(db).query_events(run_id=run_id)\n event_summary = TranslationEventLog(db).get_run_event_summary(run_id)\n\n # Get config snapshot from run\n from ....models.translate import TranslationRun\n run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n config_snapshot = run.config_snapshot if run else None\n\n return {\n **status_info,\n \"config_snapshot\": config_snapshot,\n \"config_hash\": run.config_hash if run else None,\n \"records\": records,\n \"events\": events,\n \"event_invariants\": event_summary,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_detail\n" }, { "contract_id": "download_skipped_csv", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_list_routes.py", - "start_line": 159, - "end_line": 205, + "start_line": 161, + "end_line": 207, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -25902,14 +25902,14 @@ } ], "anchor_syntax": "region", - "body": "# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv]\n# @BRIEF Download a CSV of skipped translation records from a run.\n# @RELATION DEPENDS_ON -> [TranslationRecord]\n@router.get(\"/runs/{run_id}/skipped.csv\")\nasync def download_skipped_csv(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Download skipped translation records as CSV.\"\"\"\n logger.info(f\"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}\")\n try:\n from ....models.translate import TranslationRecord\n from fastapi.responses import StreamingResponse\n import csv\n import io\n\n records = (\n db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run_id,\n TranslationRecord.status == \"SKIPPED\",\n )\n .all()\n )\n\n output = io.StringIO()\n writer = csv.writer(output)\n writer.writerow([\"id\", \"source_sql\", \"target_sql\", \"source_object_type\",\n \"source_object_id\", \"source_object_name\", \"error_message\"])\n for rec in records:\n writer.writerow([\n rec.id, rec.source_sql, rec.target_sql, rec.source_object_type,\n rec.source_object_id, rec.source_object_name, rec.error_message,\n ])\n\n output.seek(0)\n return StreamingResponse(\n iter([output.getvalue()]),\n media_type=\"text/csv\",\n headers={\"Content-Disposition\": f\"attachment; filename=skipped_{run_id}.csv\"},\n )\n except Exception as e:\n logger.error(f\"[translate_routes][download_skipped_csv] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion download_skipped_csv\n" + "body": "# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv]\n# @BRIEF Download a CSV of skipped translation records from a run.\n# @RELATION DEPENDS_ON -> [TranslationRecord]\n@router.get(\"/runs/{run_id}/skipped.csv\")\nasync def download_skipped_csv(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Download skipped translation records as CSV.\"\"\"\n log.reason(\"Download skipped CSV\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n from ....models.translate import TranslationRecord\n from fastapi.responses import StreamingResponse\n import csv\n import io\n\n records = (\n db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run_id,\n TranslationRecord.status == \"SKIPPED\",\n )\n .all()\n )\n\n output = io.StringIO()\n writer = csv.writer(output)\n writer.writerow([\"id\", \"source_sql\", \"target_sql\", \"source_object_type\",\n \"source_object_id\", \"source_object_name\", \"error_message\"])\n for rec in records:\n writer.writerow([\n rec.id, rec.source_sql, rec.target_sql, rec.source_object_type,\n rec.source_object_id, rec.source_object_name, rec.error_message,\n ])\n\n output.seek(0)\n return StreamingResponse(\n iter([output.getvalue()]),\n media_type=\"text/csv\",\n headers={\"Content-Disposition\": f\"attachment; filename=skipped_{run_id}.csv\"},\n )\n except Exception as e:\n log.explore(\"Download skipped CSV error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion download_skipped_csv\n" }, { "contract_id": "TranslateRunRoutesModule", "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_run_routes.py", "start_line": 1, - "end_line": 279, + "end_line": 281, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -25964,14 +25964,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateRunRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,runs]\n# @BRIEF Translation Run execution, history, status, records and batches routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.\n# @POST Translation run executed, manipulated, or queried. Results returned to caller.\n# @SIDE_EFFECT Creates/updates TranslationRun records in DB; spawns background threads for execution; queries Superset API.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db, SessionLocal\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.orchestrator import TranslationOrchestrator\nfrom ....plugins.translate.events import TranslationEventLog\nfrom ....models.translate import TranslationRun\n\nfrom ._router import router\nfrom ._helpers import _run_to_response\n\n\n# ============================================================\n# Translation Run / Execute\n# ============================================================\n\n# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Execute a translation job (trigger a run).\n# @PRE User has translate.job.execute permission. Job exists and preview is accepted.\n# @POST Translation run created and started in background. Run object returned.\n# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/jobs/{job_id}/run\", status_code=status.HTTP_201_CREATED)\nasync def run_translation(\n job_id: str,\n full_translation: bool = Query(False, description=\"If True, fetch ALL rows from Superset dataset instead of preview-only rows\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Execute a translation job (trigger a run).\n\n By default runs translation on preview-approved rows only.\n Set full_translation=true to fetch ALL rows from the Superset source dataset.\n \"\"\"\n logger.info(f\"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}, full={full_translation}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.start_run(job_id=job_id, is_scheduled=False)\n # Execute asynchronously in background with a FRESH session.\n # The request-scoped session from Depends(get_db) is closed after the handler returns,\n # so the background thread must create its own session to avoid \"This transaction is closed\".\n import threading\n\n def _execute_background():\n thread_db = SessionLocal()\n try:\n thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username)\n # Reload run from DB with the new session (the passed run object is detached)\n thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first()\n if thread_run:\n thread_orch.execute_run(thread_run, full_translation=full_translation)\n except Exception as e:\n logger.error(f\"[translate_routes][run_translation] Background execution error: {e}\")\n finally:\n thread_db.close()\n\n threading.Thread(target=_execute_background, daemon=True).start()\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][run_translation] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Run failed: {e}\")\n# #endregion run_translation\n\n\n# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry failed batches in a translation run.\n# @PRE User has translate.job.execute permission. Run exists and has failed batches.\n# @POST Failed batches re-executed. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry\")\nasync def retry_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry failed batches in a translation run.\"\"\"\n logger.info(f\"[translate_routes][retry_run] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_failed_batches(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][retry_run] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry failed: {e}\")\n# #endregion retry_run\n\n\n# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry the SQL insert phase for a completed run.\n# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED.\n# @POST SQL insert phase re-executed. Updated run returned.\n# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry-insert\")\nasync def retry_insert(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry the SQL insert phase for a completed run.\"\"\"\n logger.info(f\"[translate_routes][retry_insert] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_insert(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][retry_insert] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry insert failed: {e}\")\n# #endregion retry_insert\n\n\n# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Cancel a running translation.\n# @PRE User has translate.job.execute permission. Run is in RUNNING state.\n# @POST Run is cancelled. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/cancel\")\nasync def cancel_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Cancel a running translation.\"\"\"\n logger.info(f\"[translate_routes][cancel_run] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.cancel_run(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion cancel_run\n\n\n# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get run history for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/jobs/{job_id}/runs\")\nasync def get_run_history(\n job_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get run history for a translation job.\"\"\"\n logger.info(f\"[translate_routes][get_run_history] Job: {job_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n total, runs = orch.get_run_history(job_id, page=page, page_size=page_size)\n return {\"items\": runs, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_history\n\n\n# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get status and statistics for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}\")\nasync def get_run_status(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get status and statistics for a translation run.\"\"\"\n logger.info(f\"[translate_routes][get_run_status] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_status(run_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_status\n\n\n# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get paginated records for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}/records\")\nasync def get_run_records(\n run_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get paginated records for a translation run.\"\"\"\n logger.info(f\"[translate_routes][get_run_records] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_records\n\n\n# ============================================================\n# Batches\n# ============================================================\n\n# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get batches for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationBatch]\n@router.get(\"/runs/{run_id}/batches\")\nasync def get_batches(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get batches for a translation run.\"\"\"\n logger.info(f\"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}\")\n try:\n from ....models.translate import TranslationBatch\n batches = (\n db.query(TranslationBatch)\n .filter(TranslationBatch.run_id == run_id)\n .order_by(TranslationBatch.batch_index.asc())\n .all()\n )\n return [\n {\n \"id\": b.id,\n \"run_id\": b.run_id,\n \"batch_index\": b.batch_index,\n \"status\": b.status,\n \"total_records\": b.total_records or 0,\n \"successful_records\": b.successful_records or 0,\n \"failed_records\": b.failed_records or 0,\n \"started_at\": b.started_at.isoformat() if b.started_at else None,\n \"completed_at\": b.completed_at.isoformat() if b.completed_at else None,\n \"created_at\": b.created_at.isoformat() if b.created_at else None,\n }\n for b in batches\n ]\n except Exception as e:\n logger.error(f\"[translate_routes][get_batches] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_batches\n\n# #endregion TranslateRunRoutesModule\n" + "body": "# #region TranslateRunRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,runs]\n# @BRIEF Translation Run execution, history, status, records and batches routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions.\n# @POST Translation run executed, manipulated, or queried. Results returned to caller.\n# @SIDE_EFFECT Creates/updates TranslationRun records in DB; spawns background threads for execution; queries Superset API.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db, SessionLocal\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\n\nlog = MarkerLogger(\"TranslateRunRoutes\")\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.orchestrator import TranslationOrchestrator\nfrom ....plugins.translate.events import TranslationEventLog\nfrom ....models.translate import TranslationRun\n\nfrom ._router import router\nfrom ._helpers import _run_to_response\n\n\n# ============================================================\n# Translation Run / Execute\n# ============================================================\n\n# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Execute a translation job (trigger a run).\n# @PRE User has translate.job.execute permission. Job exists and preview is accepted.\n# @POST Translation run created and started in background. Run object returned.\n# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/jobs/{job_id}/run\", status_code=status.HTTP_201_CREATED)\nasync def run_translation(\n job_id: str,\n full_translation: bool = Query(False, description=\"If True, fetch ALL rows from Superset dataset instead of preview-only rows\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Execute a translation job (trigger a run).\n\n By default runs translation on preview-approved rows only.\n Set full_translation=true to fetch ALL rows from the Superset source dataset.\n \"\"\"\n log.reason(\"Run translation\", payload={\"job_id\": job_id, \"user\": current_user.username, \"full_translation\": full_translation})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.start_run(job_id=job_id, is_scheduled=False)\n # Execute asynchronously in background with a FRESH session.\n # The request-scoped session from Depends(get_db) is closed after the handler returns,\n # so the background thread must create its own session to avoid \"This transaction is closed\".\n import threading\n\n def _execute_background():\n thread_db = SessionLocal()\n try:\n thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username)\n # Reload run from DB with the new session (the passed run object is detached)\n thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first()\n if thread_run:\n thread_orch.execute_run(thread_run, full_translation=full_translation)\n except Exception as e:\n log.explore(\"Background execution error\", error=str(e))\n finally:\n thread_db.close()\n\n threading.Thread(target=_execute_background, daemon=True).start()\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Run translation error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Run failed: {e}\")\n# #endregion run_translation\n\n\n# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry failed batches in a translation run.\n# @PRE User has translate.job.execute permission. Run exists and has failed batches.\n# @POST Failed batches re-executed. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry\")\nasync def retry_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry failed batches in a translation run.\"\"\"\n log.reason(\"Retry run\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_failed_batches(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Retry run error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry failed: {e}\")\n# #endregion retry_run\n\n\n# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry the SQL insert phase for a completed run.\n# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED.\n# @POST SQL insert phase re-executed. Updated run returned.\n# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry-insert\")\nasync def retry_insert(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry the SQL insert phase for a completed run.\"\"\"\n log.reason(\"Retry insert\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_insert(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Retry insert error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry insert failed: {e}\")\n# #endregion retry_insert\n\n\n# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Cancel a running translation.\n# @PRE User has translate.job.execute permission. Run is in RUNNING state.\n# @POST Run is cancelled. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/cancel\")\nasync def cancel_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Cancel a running translation.\"\"\"\n log.reason(\"Cancel run\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.cancel_run(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion cancel_run\n\n\n# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get run history for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/jobs/{job_id}/runs\")\nasync def get_run_history(\n job_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get run history for a translation job.\"\"\"\n log.reason(\"Get run history\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n total, runs = orch.get_run_history(job_id, page=page, page_size=page_size)\n return {\"items\": runs, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_history\n\n\n# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get status and statistics for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}\")\nasync def get_run_status(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get status and statistics for a translation run.\"\"\"\n log.reason(\"Get run status\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_status(run_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_status\n\n\n# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get paginated records for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}/records\")\nasync def get_run_records(\n run_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get paginated records for a translation run.\"\"\"\n log.reason(\"Get run records\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_records\n\n\n# ============================================================\n# Batches\n# ============================================================\n\n# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get batches for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationBatch]\n@router.get(\"/runs/{run_id}/batches\")\nasync def get_batches(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get batches for a translation run.\"\"\"\n log.reason(\"Get batches\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n from ....models.translate import TranslationBatch\n batches = (\n db.query(TranslationBatch)\n .filter(TranslationBatch.run_id == run_id)\n .order_by(TranslationBatch.batch_index.asc())\n .all()\n )\n return [\n {\n \"id\": b.id,\n \"run_id\": b.run_id,\n \"batch_index\": b.batch_index,\n \"status\": b.status,\n \"total_records\": b.total_records or 0,\n \"successful_records\": b.successful_records or 0,\n \"failed_records\": b.failed_records or 0,\n \"started_at\": b.started_at.isoformat() if b.started_at else None,\n \"completed_at\": b.completed_at.isoformat() if b.completed_at else None,\n \"created_at\": b.created_at.isoformat() if b.created_at else None,\n }\n for b in batches\n ]\n except Exception as e:\n log.explore(\"Get batches error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_batches\n\n# #endregion TranslateRunRoutesModule\n" }, { "contract_id": "run_translation", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 33, - "end_line": 82, + "start_line": 35, + "end_line": 84, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26007,14 +26007,14 @@ } ], "anchor_syntax": "region", - "body": "# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Execute a translation job (trigger a run).\n# @PRE User has translate.job.execute permission. Job exists and preview is accepted.\n# @POST Translation run created and started in background. Run object returned.\n# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/jobs/{job_id}/run\", status_code=status.HTTP_201_CREATED)\nasync def run_translation(\n job_id: str,\n full_translation: bool = Query(False, description=\"If True, fetch ALL rows from Superset dataset instead of preview-only rows\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Execute a translation job (trigger a run).\n\n By default runs translation on preview-approved rows only.\n Set full_translation=true to fetch ALL rows from the Superset source dataset.\n \"\"\"\n logger.info(f\"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}, full={full_translation}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.start_run(job_id=job_id, is_scheduled=False)\n # Execute asynchronously in background with a FRESH session.\n # The request-scoped session from Depends(get_db) is closed after the handler returns,\n # so the background thread must create its own session to avoid \"This transaction is closed\".\n import threading\n\n def _execute_background():\n thread_db = SessionLocal()\n try:\n thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username)\n # Reload run from DB with the new session (the passed run object is detached)\n thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first()\n if thread_run:\n thread_orch.execute_run(thread_run, full_translation=full_translation)\n except Exception as e:\n logger.error(f\"[translate_routes][run_translation] Background execution error: {e}\")\n finally:\n thread_db.close()\n\n threading.Thread(target=_execute_background, daemon=True).start()\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][run_translation] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Run failed: {e}\")\n# #endregion run_translation\n" + "body": "# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Execute a translation job (trigger a run).\n# @PRE User has translate.job.execute permission. Job exists and preview is accepted.\n# @POST Translation run created and started in background. Run object returned.\n# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/jobs/{job_id}/run\", status_code=status.HTTP_201_CREATED)\nasync def run_translation(\n job_id: str,\n full_translation: bool = Query(False, description=\"If True, fetch ALL rows from Superset dataset instead of preview-only rows\"),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Execute a translation job (trigger a run).\n\n By default runs translation on preview-approved rows only.\n Set full_translation=true to fetch ALL rows from the Superset source dataset.\n \"\"\"\n log.reason(\"Run translation\", payload={\"job_id\": job_id, \"user\": current_user.username, \"full_translation\": full_translation})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.start_run(job_id=job_id, is_scheduled=False)\n # Execute asynchronously in background with a FRESH session.\n # The request-scoped session from Depends(get_db) is closed after the handler returns,\n # so the background thread must create its own session to avoid \"This transaction is closed\".\n import threading\n\n def _execute_background():\n thread_db = SessionLocal()\n try:\n thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username)\n # Reload run from DB with the new session (the passed run object is detached)\n thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first()\n if thread_run:\n thread_orch.execute_run(thread_run, full_translation=full_translation)\n except Exception as e:\n log.explore(\"Background execution error\", error=str(e))\n finally:\n thread_db.close()\n\n threading.Thread(target=_execute_background, daemon=True).start()\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Run translation error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Run failed: {e}\")\n# #endregion run_translation\n" }, { "contract_id": "retry_run", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 85, - "end_line": 110, + "start_line": 87, + "end_line": 112, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26050,14 +26050,14 @@ } ], "anchor_syntax": "region", - "body": "# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry failed batches in a translation run.\n# @PRE User has translate.job.execute permission. Run exists and has failed batches.\n# @POST Failed batches re-executed. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry\")\nasync def retry_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry failed batches in a translation run.\"\"\"\n logger.info(f\"[translate_routes][retry_run] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_failed_batches(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][retry_run] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry failed: {e}\")\n# #endregion retry_run\n" + "body": "# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry failed batches in a translation run.\n# @PRE User has translate.job.execute permission. Run exists and has failed batches.\n# @POST Failed batches re-executed. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry\")\nasync def retry_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry failed batches in a translation run.\"\"\"\n log.reason(\"Retry run\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_failed_batches(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Retry run error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry failed: {e}\")\n# #endregion retry_run\n" }, { "contract_id": "retry_insert", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 113, - "end_line": 138, + "start_line": 115, + "end_line": 140, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26093,14 +26093,14 @@ } ], "anchor_syntax": "region", - "body": "# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry the SQL insert phase for a completed run.\n# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED.\n# @POST SQL insert phase re-executed. Updated run returned.\n# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry-insert\")\nasync def retry_insert(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry the SQL insert phase for a completed run.\"\"\"\n logger.info(f\"[translate_routes][retry_insert] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_insert(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n logger.error(f\"[translate_routes][retry_insert] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry insert failed: {e}\")\n# #endregion retry_insert\n" + "body": "# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Retry the SQL insert phase for a completed run.\n# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED.\n# @POST SQL insert phase re-executed. Updated run returned.\n# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/retry-insert\")\nasync def retry_insert(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Retry the SQL insert phase for a completed run.\"\"\"\n log.reason(\"Retry insert\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.retry_insert(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n except Exception as e:\n log.explore(\"Retry insert error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f\"Retry insert failed: {e}\")\n# #endregion retry_insert\n" }, { "contract_id": "cancel_run", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 141, - "end_line": 163, + "start_line": 143, + "end_line": 165, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26136,14 +26136,14 @@ } ], "anchor_syntax": "region", - "body": "# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Cancel a running translation.\n# @PRE User has translate.job.execute permission. Run is in RUNNING state.\n# @POST Run is cancelled. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/cancel\")\nasync def cancel_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Cancel a running translation.\"\"\"\n logger.info(f\"[translate_routes][cancel_run] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.cancel_run(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion cancel_run\n" + "body": "# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Cancel a running translation.\n# @PRE User has translate.job.execute permission. Run is in RUNNING state.\n# @POST Run is cancelled. Updated run returned.\n# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.post(\"/runs/{run_id}/cancel\")\nasync def cancel_run(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"EXECUTE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Cancel a running translation.\"\"\"\n log.reason(\"Cancel run\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n run = orch.cancel_run(run_id)\n return _run_to_response(run)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion cancel_run\n" }, { "contract_id": "get_run_history", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 166, - "end_line": 187, + "start_line": 168, + "end_line": 189, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26176,14 +26176,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get run history for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/jobs/{job_id}/runs\")\nasync def get_run_history(\n job_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get run history for a translation job.\"\"\"\n logger.info(f\"[translate_routes][get_run_history] Job: {job_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n total, runs = orch.get_run_history(job_id, page=page, page_size=page_size)\n return {\"items\": runs, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_history\n" + "body": "# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get run history for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/jobs/{job_id}/runs\")\nasync def get_run_history(\n job_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(20, ge=1, le=100),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get run history for a translation job.\"\"\"\n log.reason(\"Get run history\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n total, runs = orch.get_run_history(job_id, page=page, page_size=page_size)\n return {\"items\": runs, \"total\": total, \"page\": page, \"page_size\": page_size}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_history\n" }, { "contract_id": "get_run_status", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 190, - "end_line": 208, + "start_line": 192, + "end_line": 210, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26216,14 +26216,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get status and statistics for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}\")\nasync def get_run_status(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get status and statistics for a translation run.\"\"\"\n logger.info(f\"[translate_routes][get_run_status] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_status(run_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_status\n" + "body": "# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get status and statistics for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}\")\nasync def get_run_status(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get status and statistics for a translation run.\"\"\"\n log.reason(\"Get run status\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_status(run_id)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_status\n" }, { "contract_id": "get_run_records", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 211, - "end_line": 232, + "start_line": 213, + "end_line": 234, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26256,14 +26256,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get paginated records for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}/records\")\nasync def get_run_records(\n run_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get paginated records for a translation run.\"\"\"\n logger.info(f\"[translate_routes][get_run_records] Run: {run_id}, User: {current_user.username}\")\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_records\n" + "body": "# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get paginated records for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n@router.get(\"/runs/{run_id}/records\")\nasync def get_run_records(\n run_id: str,\n page: int = Query(1, ge=1),\n page_size: int = Query(50, ge=1, le=500),\n status: Optional[str] = Query(None),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.history\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get paginated records for a translation run.\"\"\"\n log.reason(\"Get run records\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n orch = TranslationOrchestrator(db, config_manager, current_user.username)\n return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status)\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_run_records\n" }, { "contract_id": "get_batches", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_run_routes.py", - "start_line": 239, - "end_line": 277, + "start_line": 241, + "end_line": 279, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26296,14 +26296,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get batches for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationBatch]\n@router.get(\"/runs/{run_id}/batches\")\nasync def get_batches(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get batches for a translation run.\"\"\"\n logger.info(f\"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}\")\n try:\n from ....models.translate import TranslationBatch\n batches = (\n db.query(TranslationBatch)\n .filter(TranslationBatch.run_id == run_id)\n .order_by(TranslationBatch.batch_index.asc())\n .all()\n )\n return [\n {\n \"id\": b.id,\n \"run_id\": b.run_id,\n \"batch_index\": b.batch_index,\n \"status\": b.status,\n \"total_records\": b.total_records or 0,\n \"successful_records\": b.successful_records or 0,\n \"failed_records\": b.failed_records or 0,\n \"started_at\": b.started_at.isoformat() if b.started_at else None,\n \"completed_at\": b.completed_at.isoformat() if b.completed_at else None,\n \"created_at\": b.created_at.isoformat() if b.created_at else None,\n }\n for b in batches\n ]\n except Exception as e:\n logger.error(f\"[translate_routes][get_batches] Error: {e}\")\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_batches\n" + "body": "# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs]\n# @BRIEF Get batches for a translation run.\n# @RELATION DEPENDS_ON -> [TranslationBatch]\n@router.get(\"/runs/{run_id}/batches\")\nasync def get_batches(\n run_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.job\", \"VIEW\")),\n db: Session = Depends(get_db),\n):\n \"\"\"Get batches for a translation run.\"\"\"\n log.reason(\"Get batches\", payload={\"run_id\": run_id, \"user\": current_user.username})\n try:\n from ....models.translate import TranslationBatch\n batches = (\n db.query(TranslationBatch)\n .filter(TranslationBatch.run_id == run_id)\n .order_by(TranslationBatch.batch_index.asc())\n .all()\n )\n return [\n {\n \"id\": b.id,\n \"run_id\": b.run_id,\n \"batch_index\": b.batch_index,\n \"status\": b.status,\n \"total_records\": b.total_records or 0,\n \"successful_records\": b.successful_records or 0,\n \"failed_records\": b.failed_records or 0,\n \"started_at\": b.started_at.isoformat() if b.started_at else None,\n \"completed_at\": b.completed_at.isoformat() if b.completed_at else None,\n \"created_at\": b.created_at.isoformat() if b.created_at else None,\n }\n for b in batches\n ]\n except Exception as e:\n log.explore(\"Get batches error\", error=str(e))\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion get_batches\n" }, { "contract_id": "TranslateScheduleRoutesModule", "contract_type": "Module", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", "start_line": 1, - "end_line": 246, + "end_line": 248, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26358,14 +26358,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateScheduleRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,schedule]\n# @BRIEF Translation Schedule management routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate.schedule permissions.\n# @POST Schedule CRUD operations completed. APScheduler jobs registered/unregistered.\n# @SIDE_EFFECT Creates/updates/deletes TranslationSchedule records in DB; registers/removes jobs in APScheduler.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.logger import logger, belief_scope\nfrom ....schemas.auth import User\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.scheduler import TranslationScheduler\nfrom ....schemas.translate import ScheduleConfig, ScheduleResponse\n\nfrom ._router import router\n\n\n# ============================================================\n# Schedule\n# ============================================================\n\n# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Get the schedule configuration for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule\")\nasync def get_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][get_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"next_run_at\": schedule.next_run_at.isoformat() if schedule.next_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_schedule\n\n\n# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Set or update the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Job exists.\n# @POST Schedule created or updated. APScheduler job registered.\n# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.put(\"/jobs/{job_id}/schedule\")\nasync def set_schedule(\n job_id: str,\n payload: ScheduleConfig,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Set or update schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][set_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n # Check if schedule already exists\n from ....models.translate import TranslationSchedule\n existing = db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if existing:\n schedule = scheduler.update_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n else:\n schedule = scheduler.create_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n # Register with APScheduler via SchedulerService\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion set_schedule\n\n\n# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Enable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is enabled. APScheduler job registered.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/enable\")\nasync def enable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Enable a schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][enable_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.set_schedule_active(job_id, is_active=True)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\"status\": \"enabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion enable_schedule\n\n\n# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Disable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is disabled. APScheduler job removed.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/disable\")\nasync def disable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Disable a schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][disable_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.set_schedule_active(job_id, is_active=False)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return {\"status\": \"disabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion disable_schedule\n\n\n# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Delete the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is removed. APScheduler job removed.\n# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.delete(\"/jobs/{job_id}/schedule\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][delete_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.delete_schedule(job_id)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_schedule\n\n\n# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Preview next N execution times for a job's schedule.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule/next-executions\")\nasync def get_next_executions(\n job_id: str,\n n: int = Query(3, ge=1, le=10),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview next N executions for a job's schedule.\"\"\"\n logger.info(f\"[translate_routes][get_next_executions] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n next_times = TranslationScheduler.get_next_executions(\n schedule.cron_expression, schedule.timezone, n=n\n )\n return {\n \"job_id\": job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"next_executions\": next_times,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_next_executions\n\n# #endregion TranslateScheduleRoutesModule\n" + "body": "# #region TranslateScheduleRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,schedule]\n# @BRIEF Translation Schedule management routes.\n# @LAYER UI\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @RELATION DEPENDS_ON -> [get_current_user]\n# @RELATION DEPENDS_ON -> [get_db]\n# @PRE ConfigManager and DB session initialized. User authenticated with translate.schedule permissions.\n# @POST Schedule CRUD operations completed. APScheduler jobs registered/unregistered.\n# @SIDE_EFFECT Creates/updates/deletes TranslationSchedule records in DB; registers/removes jobs in APScheduler.\n\nfrom fastapi import APIRouter, Depends, HTTPException, status, Query\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\n\nfrom ....core.database import get_db\nfrom ....core.cot_logger import MarkerLogger\nfrom ....schemas.auth import User\n\nlog = MarkerLogger(\"TranslateScheduleRoutes\")\nfrom ....dependencies import get_current_user, has_permission, get_config_manager\nfrom ....core.config_manager import ConfigManager\nfrom ....plugins.translate.scheduler import TranslationScheduler\nfrom ....schemas.translate import ScheduleConfig, ScheduleResponse\n\nfrom ._router import router\n\n\n# ============================================================\n# Schedule\n# ============================================================\n\n# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Get the schedule configuration for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule\")\nasync def get_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get schedule for a translation job.\"\"\"\n log.reason(\"Get schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"next_run_at\": schedule.next_run_at.isoformat() if schedule.next_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_schedule\n\n\n# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Set or update the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Job exists.\n# @POST Schedule created or updated. APScheduler job registered.\n# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.put(\"/jobs/{job_id}/schedule\")\nasync def set_schedule(\n job_id: str,\n payload: ScheduleConfig,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Set or update schedule for a translation job.\"\"\"\n log.reason(\"Set schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n # Check if schedule already exists\n from ....models.translate import TranslationSchedule\n existing = db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if existing:\n schedule = scheduler.update_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n else:\n schedule = scheduler.create_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n # Register with APScheduler via SchedulerService\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion set_schedule\n\n\n# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Enable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is enabled. APScheduler job registered.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/enable\")\nasync def enable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Enable a schedule for a translation job.\"\"\"\n log.reason(\"Enable schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.set_schedule_active(job_id, is_active=True)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\"status\": \"enabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion enable_schedule\n\n\n# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Disable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is disabled. APScheduler job removed.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/disable\")\nasync def disable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Disable a schedule for a translation job.\"\"\"\n log.reason(\"Disable schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.set_schedule_active(job_id, is_active=False)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return {\"status\": \"disabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion disable_schedule\n\n\n# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Delete the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is removed. APScheduler job removed.\n# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.delete(\"/jobs/{job_id}/schedule\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete schedule for a translation job.\"\"\"\n log.reason(f\"Delete schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.delete_schedule(job_id)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_schedule\n\n\n# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Preview next N execution times for a job's schedule.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule/next-executions\")\nasync def get_next_executions(\n job_id: str,\n n: int = Query(3, ge=1, le=10),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview next N executions for a job's schedule.\"\"\"\n log.reason(f\"Get next executions\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n next_times = TranslationScheduler.get_next_executions(\n schedule.cron_expression, schedule.timezone, n=n\n )\n return {\n \"job_id\": job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"next_executions\": next_times,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_next_executions\n\n# #endregion TranslateScheduleRoutesModule\n" }, { "contract_id": "get_schedule", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", - "start_line": 31, - "end_line": 61, + "start_line": 33, + "end_line": 63, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26398,14 +26398,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Get the schedule configuration for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule\")\nasync def get_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][get_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"next_run_at\": schedule.next_run_at.isoformat() if schedule.next_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_schedule\n" + "body": "# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Get the schedule configuration for a translation job.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule\")\nasync def get_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Get schedule for a translation job.\"\"\"\n log.reason(\"Get schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"next_run_at\": schedule.next_run_at.isoformat() if schedule.next_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_schedule\n" }, { "contract_id": "set_schedule", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", - "start_line": 64, - "end_line": 124, + "start_line": 66, + "end_line": 126, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26441,14 +26441,14 @@ } ], "anchor_syntax": "region", - "body": "# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Set or update the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Job exists.\n# @POST Schedule created or updated. APScheduler job registered.\n# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.put(\"/jobs/{job_id}/schedule\")\nasync def set_schedule(\n job_id: str,\n payload: ScheduleConfig,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Set or update schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][set_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n # Check if schedule already exists\n from ....models.translate import TranslationSchedule\n existing = db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if existing:\n schedule = scheduler.update_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n else:\n schedule = scheduler.create_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n # Register with APScheduler via SchedulerService\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion set_schedule\n" + "body": "# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Set or update the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Job exists.\n# @POST Schedule created or updated. APScheduler job registered.\n# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.put(\"/jobs/{job_id}/schedule\")\nasync def set_schedule(\n job_id: str,\n payload: ScheduleConfig,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Set or update schedule for a translation job.\"\"\"\n log.reason(\"Set schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n # Check if schedule already exists\n from ....models.translate import TranslationSchedule\n existing = db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if existing:\n schedule = scheduler.update_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n else:\n schedule = scheduler.create_schedule(\n job_id,\n cron_expression=payload.cron_expression,\n timezone=payload.timezone,\n is_active=payload.is_active,\n )\n # Register with APScheduler via SchedulerService\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\n \"id\": schedule.id,\n \"job_id\": schedule.job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n \"last_run_at\": schedule.last_run_at.isoformat() if schedule.last_run_at else None,\n \"created_by\": schedule.created_by,\n \"created_at\": schedule.created_at.isoformat() if schedule.created_at else None,\n \"updated_at\": schedule.updated_at.isoformat() if schedule.updated_at else None,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))\n# #endregion set_schedule\n" }, { "contract_id": "enable_schedule", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", - "start_line": 127, - "end_line": 157, + "start_line": 129, + "end_line": 159, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26484,14 +26484,14 @@ } ], "anchor_syntax": "region", - "body": "# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Enable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is enabled. APScheduler job registered.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/enable\")\nasync def enable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Enable a schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][enable_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.set_schedule_active(job_id, is_active=True)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\"status\": \"enabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion enable_schedule\n" + "body": "# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Enable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is enabled. APScheduler job registered.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/enable\")\nasync def enable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Enable a schedule for a translation job.\"\"\"\n log.reason(\"Enable schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.set_schedule_active(job_id, is_active=True)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.add_translation_job(\n schedule_id=schedule.id,\n job_id=job_id,\n cron_expression=schedule.cron_expression,\n timezone=schedule.timezone,\n )\n return {\"status\": \"enabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion enable_schedule\n" }, { "contract_id": "disable_schedule", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", - "start_line": 160, - "end_line": 185, + "start_line": 162, + "end_line": 187, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26527,14 +26527,14 @@ } ], "anchor_syntax": "region", - "body": "# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Disable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is disabled. APScheduler job removed.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/disable\")\nasync def disable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Disable a schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][disable_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.set_schedule_active(job_id, is_active=False)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return {\"status\": \"disabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion disable_schedule\n" + "body": "# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Disable a schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is disabled. APScheduler job removed.\n# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.post(\"/jobs/{job_id}/schedule/disable\")\nasync def disable_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Disable a schedule for a translation job.\"\"\"\n log.reason(\"Disable schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.set_schedule_active(job_id, is_active=False)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return {\"status\": \"disabled\", \"job_id\": job_id}\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion disable_schedule\n" }, { "contract_id": "delete_schedule", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", - "start_line": 188, - "end_line": 213, + "start_line": 190, + "end_line": 215, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -26570,14 +26570,14 @@ } ], "anchor_syntax": "region", - "body": "# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Delete the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is removed. APScheduler job removed.\n# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.delete(\"/jobs/{job_id}/schedule\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete schedule for a translation job.\"\"\"\n logger.info(f\"[translate_routes][delete_schedule] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.delete_schedule(job_id)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_schedule\n" + "body": "# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Delete the schedule for a translation job.\n# @PRE User has translate.schedule.manage permission. Schedule exists.\n# @POST Schedule is removed. APScheduler job removed.\n# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.delete(\"/jobs/{job_id}/schedule\", status_code=status.HTTP_204_NO_CONTENT)\nasync def delete_schedule(\n job_id: str,\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"MANAGE\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Delete schedule for a translation job.\"\"\"\n log.reason(f\"Delete schedule\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n scheduler.delete_schedule(job_id)\n from ....dependencies import get_scheduler_service\n sched_svc = get_scheduler_service()\n sched_svc.remove_translation_job(schedule_id=job_id)\n return None\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion delete_schedule\n" }, { "contract_id": "get_next_executions", "contract_type": "Function", "file_path": "backend/src/api/routes/translate/_schedule_routes.py", - "start_line": 216, - "end_line": 244, + "start_line": 218, + "end_line": 246, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26610,14 +26610,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Preview next N execution times for a job's schedule.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule/next-executions\")\nasync def get_next_executions(\n job_id: str,\n n: int = Query(3, ge=1, le=10),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview next N executions for a job's schedule.\"\"\"\n logger.info(f\"[translate_routes][get_next_executions] Job: {job_id}, User: {current_user.username}\")\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n next_times = TranslationScheduler.get_next_executions(\n schedule.cron_expression, schedule.timezone, n=n\n )\n return {\n \"job_id\": job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"next_executions\": next_times,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_next_executions\n" + "body": "# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule]\n# @BRIEF Preview next N execution times for a job's schedule.\n# @RELATION DEPENDS_ON -> [TranslationScheduler]\n@router.get(\"/jobs/{job_id}/schedule/next-executions\")\nasync def get_next_executions(\n job_id: str,\n n: int = Query(3, ge=1, le=10),\n current_user: User = Depends(get_current_user),\n _ = Depends(has_permission(\"translate.schedule\", \"VIEW\")),\n db: Session = Depends(get_db),\n config_manager: ConfigManager = Depends(get_config_manager),\n):\n \"\"\"Preview next N executions for a job's schedule.\"\"\"\n log.reason(f\"Get next executions\", payload={\"job_id\": job_id, \"user\": current_user.username})\n try:\n scheduler = TranslationScheduler(db, config_manager, current_user.username)\n schedule = scheduler.get_schedule(job_id)\n next_times = TranslationScheduler.get_next_executions(\n schedule.cron_expression, schedule.timezone, n=n\n )\n return {\n \"job_id\": job_id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"next_executions\": next_times,\n }\n except ValueError as e:\n raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))\n# #endregion get_next_executions\n" }, { "contract_id": "AppModule", "contract_type": "Module", "file_path": "backend/src/app.py", "start_line": 1, - "end_line": 502, + "end_line": 514, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -26701,14 +26701,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:AppModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: app, main, entrypoint, fastapi\n# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.\n# @LAYER: UI (API)\n# @RELATION: [DEPENDS_ON] ->[AppDependencies]\n# @RELATION: [DEPENDS_ON] ->[ApiRoutesModule]\n# @INVARIANT: Only one FastAPI app instance exists per process.\n# @INVARIANT: All WebSocket connections must be properly cleaned up on disconnect.\n# @PRE: Python environment and dependencies installed; configuration database available.\n# @POST: FastAPI app instance is created, middleware configured, and routes registered.\n# @SIDE_EFFECT: Starts background scheduler and binds network ports for HTTP/WS traffic.\n# @DATA_CONTRACT: [HTTP Request | WS Message] -> [HTTP Response | JSON Log Stream]\n\nimport os\nfrom pathlib import Path\n\n# project_root is used for static files mounting\nproject_root = Path(__file__).resolve().parent.parent.parent\n\nfrom fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPException\nfrom starlette.middleware.sessions import SessionMiddleware\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.staticfiles import StaticFiles\nfrom fastapi.responses import FileResponse\nimport asyncio\n\nfrom .dependencies import get_task_manager, get_scheduler_service\nfrom .core.encryption_key import ensure_encryption_key\nfrom .core.utils.network import NetworkError\nfrom .core.logger import logger, belief_scope\nfrom .core.database import AuthSessionLocal\nfrom .core.auth.security import get_password_hash\nfrom .models.auth import User, Role\nfrom .api.routes import (\n plugins,\n tasks,\n settings,\n environments,\n mappings,\n migration,\n connections,\n git,\n storage,\n admin,\n llm,\n dashboards,\n datasets,\n reports,\n assistant,\n clean_release,\n clean_release_v2,\n profile,\n health,\n dataset_review,\n translate,\n)\nfrom .api import auth\n\n# [DEF:FastAPI_App:Global]\n# @COMPLEXITY: 3\n# @SEMANTICS: app, fastapi, instance, route-registry\n# @PURPOSE: Canonical FastAPI application instance for route, middleware, and websocket registration.\n# @RELATION: DEPENDS_ON -> [ApiRoutesModule]\n# @RELATION: BINDS_TO -> [API_Routes]\napp = FastAPI(\n title=\"Superset Tools API\",\n description=\"API for managing Superset automation tools and plugins.\",\n version=\"1.0.0\",\n)\n# [/DEF:FastAPI_App:Global]\n\n\n# [DEF:ensure_initial_admin_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Ensures initial admin user exists when bootstrap env flags are enabled.\n# @RELATION: DEPENDS_ON -> AuthRepository\ndef ensure_initial_admin_user() -> None:\n raw_flag = os.getenv(\"INITIAL_ADMIN_CREATE\", \"false\").strip().lower()\n if raw_flag not in {\"1\", \"true\", \"yes\", \"on\"}:\n return\n username = os.getenv(\"INITIAL_ADMIN_USERNAME\", \"\").strip()\n password = os.getenv(\"INITIAL_ADMIN_PASSWORD\", \"\").strip()\n if not username or not password:\n logger.warning(\n \"INITIAL_ADMIN_CREATE is enabled but INITIAL_ADMIN_USERNAME/INITIAL_ADMIN_PASSWORD is missing; skipping bootstrap.\"\n )\n return\n\n db = AuthSessionLocal()\n try:\n admin_role = db.query(Role).filter(Role.name == \"Admin\").first()\n if not admin_role:\n admin_role = Role(name=\"Admin\", description=\"System Administrator\")\n db.add(admin_role)\n db.commit()\n db.refresh(admin_role)\n\n existing_user = db.query(User).filter(User.username == username).first()\n if existing_user:\n logger.info(\n \"Initial admin bootstrap skipped: user '%s' already exists.\", username\n )\n return\n\n new_user = User(\n username=username,\n email=None,\n password_hash=get_password_hash(password),\n auth_source=\"LOCAL\",\n is_active=True,\n )\n new_user.roles.append(admin_role)\n db.add(new_user)\n db.commit()\n logger.info(\n \"Initial admin user '%s' created from environment bootstrap.\", username\n )\n except Exception as exc:\n db.rollback()\n logger.error(\"Failed to bootstrap initial admin user: %s\", exc)\n raise\n finally:\n db.close()\n\n\n# [/DEF:ensure_initial_admin_user:Function]\n\n\n# [DEF:startup_event:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Handles application startup tasks, such as starting the scheduler.\n# @RELATION: [CALLS] ->[AppDependencies]\n# @PRE: None.\n# @POST: Scheduler is started.\n# Startup event\n@app.on_event(\"startup\")\nasync def startup_event():\n with belief_scope(\"startup_event\"):\n ensure_encryption_key()\n ensure_initial_admin_user()\n scheduler = get_scheduler_service()\n scheduler.start()\n\n\n# [/DEF:startup_event:Function]\n\n\n# [DEF:shutdown_event:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Handles application shutdown tasks, such as stopping the scheduler.\n# @RELATION: [CALLS] ->[AppDependencies]\n# @PRE: None.\n# @POST: Scheduler is stopped.\n# Shutdown event\n@app.on_event(\"shutdown\")\nasync def shutdown_event():\n with belief_scope(\"shutdown_event\"):\n scheduler = get_scheduler_service()\n scheduler.stop()\n\n\n# [/DEF:shutdown_event:Function]\n\n# [DEF:app_middleware:Block]\n# @PURPOSE: Configure application-wide middleware (Session, CORS).\n# Configure Session Middleware (required by Authlib for OAuth2 flow)\nfrom .core.auth.config import auth_config\n\napp.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY)\n\n# Configure CORS\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # Adjust this in production\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n# [/DEF:app_middleware:Block]\n\n\n# [DEF:network_error_handler:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Global exception handler for NetworkError.\n# @PRE: request is a FastAPI Request object.\n# @POST: Returns 503 HTTP Exception.\n# @PARAM: request (Request) - The incoming request object.\n# @PARAM: exc (NetworkError) - The exception instance.\n@app.exception_handler(NetworkError)\nasync def network_error_handler(request: Request, exc: NetworkError):\n with belief_scope(\"network_error_handler\"):\n logger.error(f\"Network error: {exc}\")\n return HTTPException(\n status_code=503,\n detail=\"Environment unavailable. Please check if the Superset instance is running.\",\n )\n\n\n# [/DEF:network_error_handler:Function]\n\n\n# [DEF:log_requests:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Middleware to log incoming HTTP requests and their response status.\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\n# @PRE: request is a FastAPI Request object.\n# @POST: Logs request and response details.\n# @PARAM: request (Request) - The incoming request object.\n# @PARAM: call_next (Callable) - The next middleware or route handler.\n@app.middleware(\"http\")\nasync def log_requests(request: Request, call_next):\n with belief_scope(\"log_requests\"):\n # Avoid spamming logs for polling endpoints\n is_polling = request.url.path.endswith(\"/api/tasks\") and request.method == \"GET\"\n\n if not is_polling:\n logger.info(f\"Incoming request: {request.method} {request.url.path}\")\n\n try:\n response = await call_next(request)\n if not is_polling:\n logger.info(\n f\"Response status: {response.status_code} for {request.url.path}\"\n )\n return response\n except NetworkError as e:\n logger.error(f\"Network error caught in middleware: {e}\")\n raise HTTPException(\n status_code=503,\n detail=\"Environment unavailable. Please check if the Superset instance is running.\",\n )\n\n\n# [/DEF:log_requests:Function]\n\n# [DEF:API_Routes:Block]\n# @COMPLEXITY: 3\n# @PURPOSE: Register all FastAPI route groups exposed by the application entrypoint.\n# @RELATION: DEPENDS_ON -> [FastAPI_App]\n# @RELATION: DEPENDS_ON -> [Route_Group_Contracts]\n# @RELATION: DEPENDS_ON -> [AuthApi]\n# @RELATION: DEPENDS_ON -> [AdminApi]\n# @RELATION: DEPENDS_ON -> [PluginsRouter]\n# @RELATION: DEPENDS_ON -> [TasksRouter]\n# @RELATION: DEPENDS_ON -> [SettingsRouter]\n# @RELATION: DEPENDS_ON -> [ConnectionsRouter]\n# @RELATION: DEPENDS_ON -> [ReportsRouter]\n# @RELATION: DEPENDS_ON -> [LlmRoutes]\n# @RELATION: DEPENDS_ON -> [CleanReleaseV2Api]\n# Include API routes\napp.include_router(auth.router)\napp.include_router(admin.router)\napp.include_router(plugins.router, prefix=\"/api/plugins\", tags=[\"Plugins\"])\napp.include_router(tasks.router, prefix=\"/api/tasks\", tags=[\"Tasks\"])\napp.include_router(settings.router, prefix=\"/api/settings\", tags=[\"Settings\"])\napp.include_router(\n connections.router, prefix=\"/api/settings/connections\", tags=[\"Connections\"]\n)\napp.include_router(environments.router, tags=[\"Environments\"])\napp.include_router(mappings.router, prefix=\"/api/mappings\", tags=[\"Mappings\"])\napp.include_router(migration.router)\napp.include_router(git.router, prefix=\"/api/git\", tags=[\"Git\"])\napp.include_router(llm.router, prefix=\"/api/llm\", tags=[\"LLM\"])\napp.include_router(storage.router, prefix=\"/api/storage\", tags=[\"Storage\"])\napp.include_router(dashboards.router)\napp.include_router(datasets.router)\napp.include_router(reports.router)\napp.include_router(assistant.router, prefix=\"/api/assistant\", tags=[\"Assistant\"])\napp.include_router(clean_release.router)\napp.include_router(clean_release_v2.router)\napp.include_router(profile.router)\napp.include_router(dataset_review.router)\napp.include_router(health.router)\napp.include_router(translate.router)\n# [/DEF:API_Routes:Block]\n\n\n# [DEF:api.include_routers:Action]\n# @COMPLEXITY: 1\n# @PURPOSE: Registers all API routers with the FastAPI application.\n# @LAYER: API\n# @SEMANTICS: routes, registration, api\n# [/DEF:api.include_routers:Action]\n\n\n# [DEF:websocket_endpoint:Function]\n# @COMPLEXITY: 5\n# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering.\n# @RELATION: [CALLS] ->[TaskManagerPackage]\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\n# @PRE: task_id must be a valid task ID.\n# @POST: WebSocket connection is managed and logs are streamed until disconnect.\n# @SIDE_EFFECT: Subscribes to TaskManager log queue and broadcasts messages over network.\n# @DATA_CONTRACT: [task_id: str, source: str, level: str] -> [JSON log entry objects]\n# @INVARIANT: Every accepted WebSocket subscription is unsubscribed exactly once even when streaming fails or the client disconnects.\n# @UX_STATE: Connecting -> Streaming -> (Disconnected)\n#\n# @TEST_CONTRACT: WebSocketLogStreamApi ->\n# {\n# required_fields: {websocket: WebSocket, task_id: str},\n# optional_fields: {source: str, level: str},\n# invariants: [\n# \"Accepts the WebSocket connection\",\n# \"Applies source and level filters correctly to streamed logs\",\n# \"Cleans up subscriptions on disconnect\"\n# ]\n# }\n# @TEST_FIXTURE: valid_ws_connection -> {\"task_id\": \"test_1\", \"source\": \"plugin\"}\n# @TEST_EDGE: task_not_found_ws -> closes connection or sends error\n# @TEST_EDGE: empty_task_logs -> waits for new logs\n# @TEST_INVARIANT: consistent_streaming -> verifies: [valid_ws_connection]\n@app.websocket(\"/ws/logs/{task_id}\")\nasync def websocket_endpoint(\n websocket: WebSocket, task_id: str, source: str = None, level: str = None\n):\n \"\"\"\n WebSocket endpoint for real-time log streaming with optional server-side filtering.\n\n Query Parameters:\n source: Filter logs by source component (e.g., \"plugin\", \"superset_api\")\n level: Filter logs by minimum level (DEBUG, INFO, WARNING, ERROR)\n \"\"\"\n with belief_scope(\"websocket_endpoint\", f\"task_id={task_id}\"):\n await websocket.accept()\n\n source_filter = source.lower() if source else None\n level_filter = level.upper() if level else None\n level_hierarchy = {\"DEBUG\": 0, \"INFO\": 1, \"WARNING\": 2, \"ERROR\": 3}\n min_level = level_hierarchy.get(level_filter, 0) if level_filter else 0\n\n logger.reason(\n \"Accepted WebSocket log stream connection\",\n extra={\n \"task_id\": task_id,\n \"source_filter\": source_filter,\n \"level_filter\": level_filter,\n \"min_level\": min_level,\n },\n )\n\n task_manager = get_task_manager()\n queue = await task_manager.subscribe_logs(task_id)\n logger.reason(\n \"Subscribed WebSocket client to task log queue\",\n extra={\"task_id\": task_id},\n )\n\n def matches_filters(log_entry) -> bool:\n \"\"\"Check if log entry matches the filter criteria.\"\"\"\n log_source = getattr(log_entry, \"source\", None)\n if source_filter and str(log_source or \"\").lower() != source_filter:\n return False\n\n if level_filter:\n log_level = level_hierarchy.get(str(log_entry.level).upper(), 0)\n if log_level < min_level:\n return False\n\n return True\n\n try:\n logger.reason(\n \"Starting task log stream replay and live forwarding\",\n extra={\"task_id\": task_id},\n )\n\n initial_logs = task_manager.get_task_logs(task_id)\n initial_sent = 0\n for log_entry in initial_logs:\n if matches_filters(log_entry):\n log_dict = log_entry.dict()\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n await websocket.send_json(log_dict)\n initial_sent += 1\n\n logger.reflect(\n \"Initial task log replay completed\",\n extra={\n \"task_id\": task_id,\n \"replayed_logs\": initial_sent,\n \"total_available_logs\": len(initial_logs),\n },\n )\n\n task = task_manager.get_task(task_id)\n if task and task.status == \"AWAITING_INPUT\" and task.input_request:\n synthetic_log = {\n \"timestamp\": task.logs[-1].timestamp.isoformat()\n if task.logs\n else \"2024-01-01T00:00:00\",\n \"level\": \"INFO\",\n \"message\": \"Task paused for user input (Connection Re-established)\",\n \"context\": {\"input_request\": task.input_request},\n }\n await websocket.send_json(synthetic_log)\n logger.reason(\n \"Replayed awaiting-input prompt to restored WebSocket client\",\n extra={\"task_id\": task_id, \"task_status\": task.status},\n )\n\n while True:\n log_entry = await queue.get()\n\n if not matches_filters(log_entry):\n continue\n\n log_dict = log_entry.dict()\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n await websocket.send_json(log_dict)\n logger.reflect(\n \"Forwarded task log entry to WebSocket client\",\n extra={\n \"task_id\": task_id,\n \"level\": log_dict.get(\"level\"),\n },\n )\n\n if (\n \"Task completed successfully\" in log_entry.message\n or \"Task failed\" in log_entry.message\n ):\n logger.reason(\n \"Observed terminal task log entry; delaying to preserve client visibility\",\n extra={\"task_id\": task_id, \"message\": log_entry.message},\n )\n await asyncio.sleep(2)\n\n except WebSocketDisconnect:\n logger.reason(\n \"WebSocket client disconnected from task log stream\",\n extra={\"task_id\": task_id},\n )\n except Exception as exc:\n logger.explore(\n \"WebSocket log streaming encountered an unexpected failure\",\n extra={\"task_id\": task_id, \"error\": str(exc)},\n )\n raise\n finally:\n task_manager.unsubscribe_logs(task_id, queue)\n logger.reflect(\n \"Released WebSocket log queue subscription\",\n extra={\"task_id\": task_id},\n )\n\n\n# [/DEF:websocket_endpoint:Function]\n\n# [DEF:StaticFiles:Mount]\n# @COMPLEXITY: 1\n# @SEMANTICS: static, frontend, spa\n# @PURPOSE: Mounts the frontend build directory to serve static assets.\nfrontend_path = project_root / \"frontend\" / \"build\"\nif frontend_path.exists():\n app.mount(\n \"/_app\", StaticFiles(directory=str(frontend_path / \"_app\")), name=\"static\"\n )\n\n # [DEF:serve_spa:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Serves the SPA frontend for any path not matched by API routes.\n # @PRE: frontend_path exists.\n # @POST: Returns the requested file or index.html.\n @app.get(\"/{file_path:path}\", include_in_schema=False)\n async def serve_spa(file_path: str):\n with belief_scope(\"serve_spa\"):\n # Only serve SPA for non-API paths\n # API routes are registered separately and should be matched by FastAPI first\n if file_path and (\n file_path.startswith(\"api/\")\n or file_path.startswith(\"/api/\")\n or file_path == \"api\"\n ):\n # This should not happen if API routers are properly registered\n # Return 404 instead of serving HTML\n raise HTTPException(\n status_code=404, detail=f\"API endpoint not found: {file_path}\"\n )\n\n full_path = frontend_path / file_path\n if file_path and full_path.is_file():\n return FileResponse(str(full_path))\n return FileResponse(str(frontend_path / \"index.html\"))\n\n # [/DEF:serve_spa:Function]\nelse:\n # [DEF:read_root:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: A simple root endpoint to confirm that the API is running when frontend is missing.\n # @PRE: None.\n # @POST: Returns a JSON message indicating API status.\n @app.get(\"/\")\n async def read_root():\n with belief_scope(\"read_root\"):\n return {\n \"message\": \"Superset Tools API is running (Frontend build not found)\"\n }\n\n # [/DEF:read_root:Function]\n# [/DEF:StaticFiles:Mount]\n# [/DEF:AppModule:Module]\n" + "body": "# [DEF:AppModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: app, main, entrypoint, fastapi\n# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.\n# @LAYER: UI (API)\n# @RELATION: [DEPENDS_ON] ->[AppDependencies]\n# @RELATION: [DEPENDS_ON] ->[ApiRoutesModule]\n# @INVARIANT: Only one FastAPI app instance exists per process.\n# @INVARIANT: All WebSocket connections must be properly cleaned up on disconnect.\n# @PRE: Python environment and dependencies installed; configuration database available.\n# @POST: FastAPI app instance is created, middleware configured, and routes registered.\n# @SIDE_EFFECT: Starts background scheduler and binds network ports for HTTP/WS traffic.\n# @DATA_CONTRACT: [HTTP Request | WS Message] -> [HTTP Response | JSON Log Stream]\n\nimport os\nfrom pathlib import Path\n\n# project_root is used for static files mounting\nproject_root = Path(__file__).resolve().parent.parent.parent\n\nfrom fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPException\nfrom starlette.middleware.sessions import SessionMiddleware\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.staticfiles import StaticFiles\nfrom fastapi.responses import FileResponse\nimport asyncio\n\nfrom .dependencies import get_task_manager, get_scheduler_service\nfrom .core.encryption_key import ensure_encryption_key\nfrom .core.utils.network import NetworkError\nfrom .core.logger import logger, belief_scope\nfrom .core.cot_logger import MarkerLogger\n\nstartup_log = MarkerLogger(\"Startup\")\nfrom .core.database import AuthSessionLocal\nfrom .core.auth.security import get_password_hash\nfrom .models.auth import User, Role\nfrom .api.routes import (\n plugins,\n tasks,\n settings,\n environments,\n mappings,\n migration,\n connections,\n git,\n storage,\n admin,\n llm,\n dashboards,\n datasets,\n reports,\n assistant,\n clean_release,\n clean_release_v2,\n profile,\n health,\n dataset_review,\n translate,\n)\nfrom .api import auth\n\n# [DEF:FastAPI_App:Global]\n# @COMPLEXITY: 3\n# @SEMANTICS: app, fastapi, instance, route-registry\n# @PURPOSE: Canonical FastAPI application instance for route, middleware, and websocket registration.\n# @RELATION: DEPENDS_ON -> [ApiRoutesModule]\n# @RELATION: BINDS_TO -> [API_Routes]\napp = FastAPI(\n title=\"Superset Tools API\",\n description=\"API for managing Superset automation tools and plugins.\",\n version=\"1.0.0\",\n)\n# [/DEF:FastAPI_App:Global]\n\n\n# [DEF:ensure_initial_admin_user:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Ensures initial admin user exists when bootstrap env flags are enabled.\n# @RELATION: DEPENDS_ON -> AuthRepository\ndef ensure_initial_admin_user() -> None:\n raw_flag = os.getenv(\"INITIAL_ADMIN_CREATE\", \"false\").strip().lower()\n if raw_flag not in {\"1\", \"true\", \"yes\", \"on\"}:\n return\n username = os.getenv(\"INITIAL_ADMIN_USERNAME\", \"\").strip()\n password = os.getenv(\"INITIAL_ADMIN_PASSWORD\", \"\").strip()\n if not username or not password:\n logger.warning(\n \"INITIAL_ADMIN_CREATE is enabled but INITIAL_ADMIN_USERNAME/INITIAL_ADMIN_PASSWORD is missing; skipping bootstrap.\"\n )\n return\n\n db = AuthSessionLocal()\n try:\n admin_role = db.query(Role).filter(Role.name == \"Admin\").first()\n if not admin_role:\n admin_role = Role(name=\"Admin\", description=\"System Administrator\")\n db.add(admin_role)\n db.commit()\n db.refresh(admin_role)\n\n existing_user = db.query(User).filter(User.username == username).first()\n if existing_user:\n logger.info(\n \"Initial admin bootstrap skipped: user '%s' already exists.\", username\n )\n return\n\n new_user = User(\n username=username,\n email=None,\n password_hash=get_password_hash(password),\n auth_source=\"LOCAL\",\n is_active=True,\n )\n new_user.roles.append(admin_role)\n db.add(new_user)\n db.commit()\n logger.info(\n \"Initial admin user '%s' created from environment bootstrap.\", username\n )\n except Exception as exc:\n db.rollback()\n logger.error(\"Failed to bootstrap initial admin user: %s\", exc)\n raise\n finally:\n db.close()\n\n\n# [/DEF:ensure_initial_admin_user:Function]\n\n\n# [DEF:startup_event:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Handles application startup tasks, such as starting the scheduler.\n# @RELATION: [CALLS] ->[AppDependencies]\n# @PRE: None.\n# @POST: Scheduler is started.\n# Startup event\n@app.on_event(\"startup\")\nasync def startup_event():\n startup_log.reason(\"Enter startup event\")\n ensure_encryption_key()\n ensure_initial_admin_user()\n scheduler = get_scheduler_service()\n startup_log.reflect(\"Application startup completed\")\n scheduler.start()\n\n\n# [/DEF:startup_event:Function]\n\n\n# [DEF:shutdown_event:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Handles application shutdown tasks, such as stopping the scheduler.\n# @RELATION: [CALLS] ->[AppDependencies]\n# @PRE: None.\n# @POST: Scheduler is stopped.\n# Shutdown event\n@app.on_event(\"shutdown\")\nasync def shutdown_event():\n with belief_scope(\"shutdown_event\"):\n scheduler = get_scheduler_service()\n scheduler.stop()\n\n\n# [/DEF:shutdown_event:Function]\n\n# [DEF:app_middleware:Block]\n# @PURPOSE: Configure application-wide middleware (Session, CORS, TraceContext).\n# Configure Session Middleware (required by Authlib for OAuth2 flow)\nfrom .core.auth.config import auth_config\n\napp.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY)\n\n# Configure CORS\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # Adjust this in production\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n\n# Configure Trace Context Middleware (seeds trace_id per request)\nfrom .core.middleware.trace import TraceContextMiddleware\n\napp.add_middleware(TraceContextMiddleware)\n# [/DEF:app_middleware:Block]\n\n\n# [DEF:network_error_handler:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Global exception handler for NetworkError.\n# @PRE: request is a FastAPI Request object.\n# @POST: Returns 503 HTTP Exception.\n# @PARAM: request (Request) - The incoming request object.\n# @PARAM: exc (NetworkError) - The exception instance.\neh_log = MarkerLogger(\"ExceptionHandler\")\n\n\n@app.exception_handler(NetworkError)\nasync def network_error_handler(request: Request, exc: NetworkError):\n with belief_scope(\"network_error_handler\"):\n eh_log.explore(\"Unhandled NetworkError in request\", error=str(exc), payload={\"url\": str(request.url)})\n return HTTPException(\n status_code=503,\n detail=\"Environment unavailable. Please check if the Superset instance is running.\",\n )\n\n\n# [/DEF:network_error_handler:Function]\n\n\n# [DEF:log_requests:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Middleware to log incoming HTTP requests and their response status.\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\n# @PRE: request is a FastAPI Request object.\n# @POST: Logs request and response details.\n# @PARAM: request (Request) - The incoming request object.\n# @PARAM: call_next (Callable) - The next middleware or route handler.\n@app.middleware(\"http\")\nasync def log_requests(request: Request, call_next):\n with belief_scope(\"log_requests\"):\n # Avoid spamming logs for polling endpoints\n is_polling = request.url.path.endswith(\"/api/tasks\") and request.method == \"GET\"\n\n if not is_polling:\n logger.info(f\"Incoming request: {request.method} {request.url.path}\")\n\n try:\n response = await call_next(request)\n if not is_polling:\n logger.info(\n f\"Response status: {response.status_code} for {request.url.path}\"\n )\n return response\n except NetworkError as e:\n logger.error(f\"Network error caught in middleware: {e}\")\n raise HTTPException(\n status_code=503,\n detail=\"Environment unavailable. Please check if the Superset instance is running.\",\n )\n\n\n# [/DEF:log_requests:Function]\n\n# [DEF:API_Routes:Block]\n# @COMPLEXITY: 3\n# @PURPOSE: Register all FastAPI route groups exposed by the application entrypoint.\n# @RELATION: DEPENDS_ON -> [FastAPI_App]\n# @RELATION: DEPENDS_ON -> [Route_Group_Contracts]\n# @RELATION: DEPENDS_ON -> [AuthApi]\n# @RELATION: DEPENDS_ON -> [AdminApi]\n# @RELATION: DEPENDS_ON -> [PluginsRouter]\n# @RELATION: DEPENDS_ON -> [TasksRouter]\n# @RELATION: DEPENDS_ON -> [SettingsRouter]\n# @RELATION: DEPENDS_ON -> [ConnectionsRouter]\n# @RELATION: DEPENDS_ON -> [ReportsRouter]\n# @RELATION: DEPENDS_ON -> [LlmRoutes]\n# @RELATION: DEPENDS_ON -> [CleanReleaseV2Api]\n# Include API routes\napp.include_router(auth.router)\napp.include_router(admin.router)\napp.include_router(plugins.router, prefix=\"/api/plugins\", tags=[\"Plugins\"])\napp.include_router(tasks.router, prefix=\"/api/tasks\", tags=[\"Tasks\"])\napp.include_router(settings.router, prefix=\"/api/settings\", tags=[\"Settings\"])\napp.include_router(\n connections.router, prefix=\"/api/settings/connections\", tags=[\"Connections\"]\n)\napp.include_router(environments.router, tags=[\"Environments\"])\napp.include_router(mappings.router, prefix=\"/api/mappings\", tags=[\"Mappings\"])\napp.include_router(migration.router)\napp.include_router(git.router, prefix=\"/api/git\", tags=[\"Git\"])\napp.include_router(llm.router, prefix=\"/api/llm\", tags=[\"LLM\"])\napp.include_router(storage.router, prefix=\"/api/storage\", tags=[\"Storage\"])\napp.include_router(dashboards.router)\napp.include_router(datasets.router)\napp.include_router(reports.router)\napp.include_router(assistant.router, prefix=\"/api/assistant\", tags=[\"Assistant\"])\napp.include_router(clean_release.router)\napp.include_router(clean_release_v2.router)\napp.include_router(profile.router)\napp.include_router(dataset_review.router)\napp.include_router(health.router)\napp.include_router(translate.router)\n# [/DEF:API_Routes:Block]\n\n\n# [DEF:api.include_routers:Action]\n# @COMPLEXITY: 1\n# @PURPOSE: Registers all API routers with the FastAPI application.\n# @LAYER: API\n# @SEMANTICS: routes, registration, api\n# [/DEF:api.include_routers:Action]\n\n\n# [DEF:websocket_endpoint:Function]\n# @COMPLEXITY: 5\n# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering.\n# @RELATION: [CALLS] ->[TaskManagerPackage]\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\n# @PRE: task_id must be a valid task ID.\n# @POST: WebSocket connection is managed and logs are streamed until disconnect.\n# @SIDE_EFFECT: Subscribes to TaskManager log queue and broadcasts messages over network.\n# @DATA_CONTRACT: [task_id: str, source: str, level: str] -> [JSON log entry objects]\n# @INVARIANT: Every accepted WebSocket subscription is unsubscribed exactly once even when streaming fails or the client disconnects.\n# @UX_STATE: Connecting -> Streaming -> (Disconnected)\n#\n# @TEST_CONTRACT: WebSocketLogStreamApi ->\n# {\n# required_fields: {websocket: WebSocket, task_id: str},\n# optional_fields: {source: str, level: str},\n# invariants: [\n# \"Accepts the WebSocket connection\",\n# \"Applies source and level filters correctly to streamed logs\",\n# \"Cleans up subscriptions on disconnect\"\n# ]\n# }\n# @TEST_FIXTURE: valid_ws_connection -> {\"task_id\": \"test_1\", \"source\": \"plugin\"}\n# @TEST_EDGE: task_not_found_ws -> closes connection or sends error\n# @TEST_EDGE: empty_task_logs -> waits for new logs\n# @TEST_INVARIANT: consistent_streaming -> verifies: [valid_ws_connection]\n@app.websocket(\"/ws/logs/{task_id}\")\nasync def websocket_endpoint(\n websocket: WebSocket, task_id: str, source: str = None, level: str = None\n):\n \"\"\"\n WebSocket endpoint for real-time log streaming with optional server-side filtering.\n\n Query Parameters:\n source: Filter logs by source component (e.g., \"plugin\", \"superset_api\")\n level: Filter logs by minimum level (DEBUG, INFO, WARNING, ERROR)\n \"\"\"\n with belief_scope(\"websocket_endpoint\", f\"task_id={task_id}\"):\n await websocket.accept()\n\n source_filter = source.lower() if source else None\n level_filter = level.upper() if level else None\n level_hierarchy = {\"DEBUG\": 0, \"INFO\": 1, \"WARNING\": 2, \"ERROR\": 3}\n min_level = level_hierarchy.get(level_filter, 0) if level_filter else 0\n\n logger.reason(\n \"Accepted WebSocket log stream connection\",\n extra={\n \"task_id\": task_id,\n \"source_filter\": source_filter,\n \"level_filter\": level_filter,\n \"min_level\": min_level,\n },\n )\n\n task_manager = get_task_manager()\n queue = await task_manager.subscribe_logs(task_id)\n logger.reason(\n \"Subscribed WebSocket client to task log queue\",\n extra={\"task_id\": task_id},\n )\n\n def matches_filters(log_entry) -> bool:\n \"\"\"Check if log entry matches the filter criteria.\"\"\"\n log_source = getattr(log_entry, \"source\", None)\n if source_filter and str(log_source or \"\").lower() != source_filter:\n return False\n\n if level_filter:\n log_level = level_hierarchy.get(str(log_entry.level).upper(), 0)\n if log_level < min_level:\n return False\n\n return True\n\n try:\n logger.reason(\n \"Starting task log stream replay and live forwarding\",\n extra={\"task_id\": task_id},\n )\n\n initial_logs = task_manager.get_task_logs(task_id)\n initial_sent = 0\n for log_entry in initial_logs:\n if matches_filters(log_entry):\n log_dict = log_entry.dict()\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n await websocket.send_json(log_dict)\n initial_sent += 1\n\n logger.reflect(\n \"Initial task log replay completed\",\n extra={\n \"task_id\": task_id,\n \"replayed_logs\": initial_sent,\n \"total_available_logs\": len(initial_logs),\n },\n )\n\n task = task_manager.get_task(task_id)\n if task and task.status == \"AWAITING_INPUT\" and task.input_request:\n synthetic_log = {\n \"timestamp\": task.logs[-1].timestamp.isoformat()\n if task.logs\n else \"2024-01-01T00:00:00\",\n \"level\": \"INFO\",\n \"message\": \"Task paused for user input (Connection Re-established)\",\n \"context\": {\"input_request\": task.input_request},\n }\n await websocket.send_json(synthetic_log)\n logger.reason(\n \"Replayed awaiting-input prompt to restored WebSocket client\",\n extra={\"task_id\": task_id, \"task_status\": task.status},\n )\n\n while True:\n log_entry = await queue.get()\n\n if not matches_filters(log_entry):\n continue\n\n log_dict = log_entry.dict()\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n await websocket.send_json(log_dict)\n logger.reflect(\n \"Forwarded task log entry to WebSocket client\",\n extra={\n \"task_id\": task_id,\n \"level\": log_dict.get(\"level\"),\n },\n )\n\n if (\n \"Task completed successfully\" in log_entry.message\n or \"Task failed\" in log_entry.message\n ):\n logger.reason(\n \"Observed terminal task log entry; delaying to preserve client visibility\",\n extra={\"task_id\": task_id, \"message\": log_entry.message},\n )\n await asyncio.sleep(2)\n\n except WebSocketDisconnect:\n logger.reason(\n \"WebSocket client disconnected from task log stream\",\n extra={\"task_id\": task_id},\n )\n except Exception as exc:\n logger.explore(\n \"WebSocket log streaming encountered an unexpected failure\",\n extra={\"task_id\": task_id, \"error\": str(exc)},\n )\n raise\n finally:\n task_manager.unsubscribe_logs(task_id, queue)\n logger.reflect(\n \"Released WebSocket log queue subscription\",\n extra={\"task_id\": task_id},\n )\n\n\n# [/DEF:websocket_endpoint:Function]\n\n# [DEF:StaticFiles:Mount]\n# @COMPLEXITY: 1\n# @SEMANTICS: static, frontend, spa\n# @PURPOSE: Mounts the frontend build directory to serve static assets.\nfrontend_path = project_root / \"frontend\" / \"build\"\nif frontend_path.exists():\n app.mount(\n \"/_app\", StaticFiles(directory=str(frontend_path / \"_app\")), name=\"static\"\n )\n\n # [DEF:serve_spa:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Serves the SPA frontend for any path not matched by API routes.\n # @PRE: frontend_path exists.\n # @POST: Returns the requested file or index.html.\n @app.get(\"/{file_path:path}\", include_in_schema=False)\n async def serve_spa(file_path: str):\n with belief_scope(\"serve_spa\"):\n # Only serve SPA for non-API paths\n # API routes are registered separately and should be matched by FastAPI first\n if file_path and (\n file_path.startswith(\"api/\")\n or file_path.startswith(\"/api/\")\n or file_path == \"api\"\n ):\n # This should not happen if API routers are properly registered\n # Return 404 instead of serving HTML\n raise HTTPException(\n status_code=404, detail=f\"API endpoint not found: {file_path}\"\n )\n\n full_path = frontend_path / file_path\n if file_path and full_path.is_file():\n return FileResponse(str(full_path))\n return FileResponse(str(frontend_path / \"index.html\"))\n\n # [/DEF:serve_spa:Function]\nelse:\n # [DEF:read_root:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: A simple root endpoint to confirm that the API is running when frontend is missing.\n # @PRE: None.\n # @POST: Returns a JSON message indicating API status.\n @app.get(\"/\")\n async def read_root():\n with belief_scope(\"read_root\"):\n return {\n \"message\": \"Superset Tools API is running (Frontend build not found)\"\n }\n\n # [/DEF:read_root:Function]\n# [/DEF:StaticFiles:Mount]\n# [/DEF:AppModule:Module]\n" }, { "contract_id": "FastAPI_App", "contract_type": "Global", "file_path": "backend/src/app.py", - "start_line": 60, - "end_line": 71, + "start_line": 63, + "end_line": 74, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26822,8 +26822,8 @@ "contract_id": "ensure_initial_admin_user", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 74, - "end_line": 127, + "start_line": 77, + "end_line": 130, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26846,8 +26846,8 @@ "contract_id": "startup_event", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 130, - "end_line": 146, + "start_line": 133, + "end_line": 150, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26902,14 +26902,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:startup_event:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Handles application startup tasks, such as starting the scheduler.\n# @RELATION: [CALLS] ->[AppDependencies]\n# @PRE: None.\n# @POST: Scheduler is started.\n# Startup event\n@app.on_event(\"startup\")\nasync def startup_event():\n with belief_scope(\"startup_event\"):\n ensure_encryption_key()\n ensure_initial_admin_user()\n scheduler = get_scheduler_service()\n scheduler.start()\n\n\n# [/DEF:startup_event:Function]\n" + "body": "# [DEF:startup_event:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Handles application startup tasks, such as starting the scheduler.\n# @RELATION: [CALLS] ->[AppDependencies]\n# @PRE: None.\n# @POST: Scheduler is started.\n# Startup event\n@app.on_event(\"startup\")\nasync def startup_event():\n startup_log.reason(\"Enter startup event\")\n ensure_encryption_key()\n ensure_initial_admin_user()\n scheduler = get_scheduler_service()\n startup_log.reflect(\"Application startup completed\")\n scheduler.start()\n\n\n# [/DEF:startup_event:Function]\n" }, { "contract_id": "shutdown_event", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 149, - "end_line": 163, + "start_line": 153, + "end_line": 167, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -26970,24 +26970,24 @@ "contract_id": "app_middleware", "contract_type": "Block", "file_path": "backend/src/app.py", - "start_line": 165, - "end_line": 180, + "start_line": 169, + "end_line": 189, "tier": "TIER_1", "complexity": 1, "metadata": { - "PURPOSE": "Configure application-wide middleware (Session, CORS)." + "PURPOSE": "Configure application-wide middleware (Session, CORS, TraceContext)." }, "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:app_middleware:Block]\n# @PURPOSE: Configure application-wide middleware (Session, CORS).\n# Configure Session Middleware (required by Authlib for OAuth2 flow)\nfrom .core.auth.config import auth_config\n\napp.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY)\n\n# Configure CORS\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # Adjust this in production\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n# [/DEF:app_middleware:Block]\n" + "body": "# [DEF:app_middleware:Block]\n# @PURPOSE: Configure application-wide middleware (Session, CORS, TraceContext).\n# Configure Session Middleware (required by Authlib for OAuth2 flow)\nfrom .core.auth.config import auth_config\n\napp.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY)\n\n# Configure CORS\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # Adjust this in production\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n\n# Configure Trace Context Middleware (seeds trace_id per request)\nfrom .core.middleware.trace import TraceContextMiddleware\n\napp.add_middleware(TraceContextMiddleware)\n# [/DEF:app_middleware:Block]\n" }, { "contract_id": "network_error_handler", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 183, - "end_line": 200, + "start_line": 192, + "end_line": 212, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -27025,14 +27025,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:network_error_handler:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Global exception handler for NetworkError.\n# @PRE: request is a FastAPI Request object.\n# @POST: Returns 503 HTTP Exception.\n# @PARAM: request (Request) - The incoming request object.\n# @PARAM: exc (NetworkError) - The exception instance.\n@app.exception_handler(NetworkError)\nasync def network_error_handler(request: Request, exc: NetworkError):\n with belief_scope(\"network_error_handler\"):\n logger.error(f\"Network error: {exc}\")\n return HTTPException(\n status_code=503,\n detail=\"Environment unavailable. Please check if the Superset instance is running.\",\n )\n\n\n# [/DEF:network_error_handler:Function]\n" + "body": "# [DEF:network_error_handler:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Global exception handler for NetworkError.\n# @PRE: request is a FastAPI Request object.\n# @POST: Returns 503 HTTP Exception.\n# @PARAM: request (Request) - The incoming request object.\n# @PARAM: exc (NetworkError) - The exception instance.\neh_log = MarkerLogger(\"ExceptionHandler\")\n\n\n@app.exception_handler(NetworkError)\nasync def network_error_handler(request: Request, exc: NetworkError):\n with belief_scope(\"network_error_handler\"):\n eh_log.explore(\"Unhandled NetworkError in request\", error=str(exc), payload={\"url\": str(request.url)})\n return HTTPException(\n status_code=503,\n detail=\"Environment unavailable. Please check if the Superset instance is running.\",\n )\n\n\n# [/DEF:network_error_handler:Function]\n" }, { "contract_id": "log_requests", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 203, - "end_line": 235, + "start_line": 215, + "end_line": 247, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -27100,8 +27100,8 @@ "contract_id": "API_Routes", "contract_type": "Block", "file_path": "backend/src/app.py", - "start_line": 237, - "end_line": 276, + "start_line": 249, + "end_line": 288, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -27184,8 +27184,8 @@ "contract_id": "api.include_routers", "contract_type": "Action", "file_path": "backend/src/app.py", - "start_line": 279, - "end_line": 284, + "start_line": 291, + "end_line": 296, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -27311,8 +27311,8 @@ "contract_id": "websocket_endpoint", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 287, - "end_line": 448, + "start_line": 299, + "end_line": 460, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -27389,8 +27389,8 @@ "contract_id": "StaticFiles", "contract_type": "Mount", "file_path": "backend/src/app.py", - "start_line": 450, - "end_line": 501, + "start_line": 462, + "end_line": 513, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -27481,8 +27481,8 @@ "contract_id": "serve_spa", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 460, - "end_line": 486, + "start_line": 472, + "end_line": 498, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -27519,8 +27519,8 @@ "contract_id": "read_root", "contract_type": "Function", "file_path": "backend/src/app.py", - "start_line": 488, - "end_line": 500, + "start_line": 500, + "end_line": 512, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31733,7 +31733,7 @@ "contract_type": "Module", "file_path": "backend/src/core/config_manager.py", "start_line": 1, - "end_line": 598, + "end_line": 576, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -31873,14 +31873,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:ConfigManager:Module]\n#\n# @COMPLEXITY: 5\n# @SEMANTICS: config, manager, persistence, migration, postgresql\n# @PURPOSE: Manages application configuration persistence in DB with one-time migration from legacy JSON.\n# @LAYER: Domain\n# @PRE: Database schema for AppConfigRecord must be initialized.\n# @POST: Configuration is loaded into memory and logger is configured.\n# @SIDE_EFFECT: Performs DB I/O and may update global logging level.\n# @DATA_CONTRACT: Input[json, record] -> Model[AppConfig]\n# @INVARIANT: Configuration must always be representable by AppConfig and persisted under global record id.\n# @RELATION: [DEPENDS_ON] ->[AppConfig]\n# @RELATION: [DEPENDS_ON] ->[SessionLocal]\n# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]\n# @RELATION: [CALLS] ->[logger]\n# @RELATION: [CALLS] ->[configure_logger]\n#\nimport json\nimport os\nfrom pathlib import Path\nfrom typing import Any, Optional, List\n\nfrom sqlalchemy.orm import Session\n\nfrom .config_models import AppConfig, Environment, GlobalSettings\nfrom .database import SessionLocal\nfrom ..models.config import AppConfigRecord\nfrom ..models.mapping import Environment as EnvironmentRecord\nfrom .logger import logger, configure_logger, belief_scope\n\n\n# [DEF:ConfigManager:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Handles application configuration load, validation, mutation, and persistence lifecycle.\n# @PRE: Database is accessible and AppConfigRecord schema is loaded.\n# @POST: Configuration state is synchronized between memory and database.\n# @SIDE_EFFECT: Performs DB I/O, OS path validation, and logger reconfiguration.\nclass ConfigManager:\n # [DEF:__init__:Function]\n # @PURPOSE: Initialize manager state from persisted or migrated configuration.\n # @PRE: config_path is a non-empty string path.\n # @POST: self.config is initialized as AppConfig and logger is configured.\n # @SIDE_EFFECT: Reads config sources and updates logging configuration.\n # @DATA_CONTRACT: Input(str config_path) -> Output(None; self.config: AppConfig)\n def __init__(self, config_path: str = \"config.json\"):\n with belief_scope(\"ConfigManager.__init__\"):\n if not isinstance(config_path, str) or not config_path:\n logger.explore(\n \"Invalid config_path provided\", extra={\"path\": config_path}\n )\n raise ValueError(\"config_path must be a non-empty string\")\n\n logger.reason(f\"Initializing ConfigManager with legacy path: {config_path}\")\n\n self.config_path = Path(config_path)\n self.raw_payload: dict[str, Any] = {}\n self.config: AppConfig = self._load_config()\n\n configure_logger(self.config.settings.logging)\n\n if not isinstance(self.config, AppConfig):\n logger.explore(\n \"Config loading resulted in invalid type\",\n extra={\"type\": type(self.config)},\n )\n raise TypeError(\"self.config must be an instance of AppConfig\")\n\n logger.reflect(\"ConfigManager initialization complete\")\n\n # [/DEF:__init__:Function]\n\n # [DEF:_apply_features_from_env:Function]\n # @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config.\n # @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place.\n # @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth.\n @staticmethod\n def _apply_features_from_env(settings: GlobalSettings) -> None:\n with belief_scope(\"ConfigManager._apply_features_from_env\"):\n dataset_review_env = os.getenv(\"FEATURES__DATASET_REVIEW\")\n if dataset_review_env is not None:\n parsed = dataset_review_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.dataset_review = parsed\n logger.reason(\n \"Applied FEATURES__DATASET_REVIEW from env\",\n extra={\"value\": parsed, \"raw\": dataset_review_env},\n )\n\n health_monitor_env = os.getenv(\"FEATURES__HEALTH_MONITOR\")\n if health_monitor_env is not None:\n parsed = health_monitor_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.health_monitor = parsed\n logger.reason(\n \"Applied FEATURES__HEALTH_MONITOR from env\",\n extra={\"value\": parsed, \"raw\": health_monitor_env},\n )\n\n # [/DEF:_apply_features_from_env:Function]\n\n # [DEF:_default_config:Function]\n # @PURPOSE: Build default application configuration fallback.\n def _default_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._default_config\"):\n logger.reason(\"Building default AppConfig fallback\")\n config = AppConfig(environments=[], settings=GlobalSettings())\n self._apply_features_from_env(config.settings)\n return config\n\n # [/DEF:_default_config:Function]\n\n # [DEF:_sync_raw_payload_from_config:Function]\n # @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.\n def _sync_raw_payload_from_config(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._sync_raw_payload_from_config\"):\n typed_payload = self.config.model_dump()\n merged_payload = dict(self.raw_payload or {})\n merged_payload[\"environments\"] = typed_payload.get(\"environments\", [])\n merged_payload[\"settings\"] = typed_payload.get(\"settings\", {})\n self.raw_payload = merged_payload\n logger.reason(\n \"Synchronized raw payload from typed config\",\n extra={\n \"environments_count\": len(\n merged_payload.get(\"environments\", []) or []\n ),\n \"has_settings\": \"settings\" in merged_payload,\n \"extra_sections\": sorted(\n key\n for key in merged_payload.keys()\n if key not in {\"environments\", \"settings\"}\n ),\n },\n )\n return merged_payload\n\n # [/DEF:_sync_raw_payload_from_config:Function]\n\n # [DEF:_load_from_legacy_file:Function]\n # @PURPOSE: Load legacy JSON configuration for migration fallback path.\n def _load_from_legacy_file(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._load_from_legacy_file\"):\n if not self.config_path.exists():\n logger.reason(\n \"Legacy config file not found; using default payload\",\n extra={\"path\": str(self.config_path)},\n )\n return {}\n\n logger.reason(\n \"Loading legacy config file\", extra={\"path\": str(self.config_path)}\n )\n with self.config_path.open(\"r\", encoding=\"utf-8\") as fh:\n payload = json.load(fh)\n\n if not isinstance(payload, dict):\n logger.explore(\n \"Legacy config payload is not a JSON object\",\n extra={\n \"path\": str(self.config_path),\n \"type\": type(payload).__name__,\n },\n )\n raise ValueError(\"Legacy config payload must be a JSON object\")\n\n logger.reason(\n \"Legacy config file loaded successfully\",\n extra={\"path\": str(self.config_path), \"keys\": sorted(payload.keys())},\n )\n return payload\n\n # [/DEF:_load_from_legacy_file:Function]\n\n # [DEF:_get_record:Function]\n # @PURPOSE: Resolve global configuration record from DB.\n def _get_record(self, session: Session) -> Optional[AppConfigRecord]:\n with belief_scope(\"ConfigManager._get_record\"):\n record = (\n session.query(AppConfigRecord)\n .filter(AppConfigRecord.id == \"global\")\n .first()\n )\n logger.reason(\n \"Resolved app config record\", extra={\"exists\": record is not None}\n )\n return record\n\n # [/DEF:_get_record:Function]\n\n # [DEF:_load_config:Function]\n # @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.\n def _load_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._load_config\"):\n session = SessionLocal()\n try:\n record = self._get_record(session)\n if record and isinstance(record.payload, dict):\n logger.reason(\n \"Loading configuration from database\",\n extra={\"record_id\": record.id},\n )\n self.raw_payload = dict(record.payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._sync_environment_records(session, config)\n session.commit()\n logger.reason(\n \"Database configuration validated successfully\",\n extra={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n return config\n\n logger.reason(\n \"Database configuration record missing; attempting legacy file migration\",\n extra={\"legacy_path\": str(self.config_path)},\n )\n legacy_payload = self._load_from_legacy_file()\n\n if legacy_payload:\n self.raw_payload = dict(legacy_payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._apply_features_from_env(config.settings)\n logger.reason(\n \"Legacy payload validated; persisting migrated configuration to database\",\n extra={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n self._save_config_to_db(config, session=session)\n return config\n\n logger.reason(\n \"No persisted config found; falling back to default configuration\"\n )\n config = self._default_config()\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config, session=session)\n return config\n except (json.JSONDecodeError, TypeError, ValueError) as exc:\n logger.explore(\n \"Recoverable config load failure; falling back to default configuration\",\n extra={\"error\": str(exc), \"legacy_path\": str(self.config_path)},\n )\n config = self._default_config()\n self.raw_payload = config.model_dump()\n return config\n except Exception as exc:\n logger.explore(\n \"Critical config load failure; re-raising persistence or validation error\",\n extra={\"error\": str(exc)},\n )\n raise\n finally:\n session.close()\n\n # [/DEF:_load_config:Function]\n\n # [DEF:_sync_environment_records:Function]\n # @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models.\n def _sync_environment_records(self, session: Session, config: AppConfig) -> None:\n with belief_scope(\"ConfigManager._sync_environment_records\"):\n configured_envs = list(config.environments or [])\n persisted_records = session.query(EnvironmentRecord).all()\n persisted_by_id = {\n str(record.id or \"\").strip(): record for record in persisted_records\n }\n\n for environment in configured_envs:\n normalized_id = str(environment.id or \"\").strip()\n if not normalized_id:\n continue\n\n display_name = (\n str(environment.name or normalized_id).strip() or normalized_id\n )\n normalized_url = str(environment.url or \"\").strip()\n credentials_id = (\n str(environment.username or \"\").strip() or normalized_id\n )\n\n record = persisted_by_id.get(normalized_id)\n if record is None:\n logger.reason(\n \"Creating relational environment record from typed config\",\n extra={\n \"environment_id\": normalized_id,\n \"environment_name\": display_name,\n },\n )\n session.add(\n EnvironmentRecord(\n id=normalized_id,\n name=display_name,\n url=normalized_url,\n credentials_id=credentials_id,\n )\n )\n continue\n\n record.name = display_name\n record.url = normalized_url\n record.credentials_id = credentials_id\n\n # [/DEF:_sync_environment_records:Function]\n\n # [DEF:_save_config_to_db:Function]\n # @PURPOSE: Persist provided AppConfig into the global DB configuration record.\n def _save_config_to_db(\n self, config: AppConfig, session: Optional[Session] = None\n ) -> None:\n with belief_scope(\"ConfigManager._save_config_to_db\"):\n owns_session = session is None\n db = session or SessionLocal()\n try:\n self.config = config\n payload = self._sync_raw_payload_from_config()\n record = self._get_record(db)\n if record is None:\n logger.reason(\"Creating new global app config record\")\n record = AppConfigRecord(id=\"global\", payload=payload)\n db.add(record)\n else:\n logger.reason(\n \"Updating existing global app config record\",\n extra={\"record_id\": record.id},\n )\n record.payload = payload\n\n self._sync_environment_records(db, config)\n\n db.commit()\n logger.reason(\n \"Configuration persisted to database\",\n extra={\n \"environments_count\": len(\n payload.get(\"environments\", []) or []\n ),\n \"payload_keys\": sorted(payload.keys()),\n },\n )\n except Exception:\n db.rollback()\n logger.explore(\"Database save failed; transaction rolled back\")\n raise\n finally:\n if owns_session:\n db.close()\n\n # [/DEF:_save_config_to_db:Function]\n\n # [DEF:save:Function]\n # @PURPOSE: Persist current in-memory configuration state.\n def save(self) -> None:\n with belief_scope(\"ConfigManager.save\"):\n logger.reason(\"Persisting current in-memory configuration\")\n self._save_config_to_db(self.config)\n\n # [/DEF:save:Function]\n\n # [DEF:get_config:Function]\n # @PURPOSE: Return current in-memory configuration snapshot.\n def get_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager.get_config\"):\n return self.config\n\n # [/DEF:get_config:Function]\n\n # [DEF:get_payload:Function]\n # @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema.\n def get_payload(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager.get_payload\"):\n return self._sync_raw_payload_from_config()\n\n # [/DEF:get_payload:Function]\n\n # [DEF:save_config:Function]\n # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.\n def save_config(self, config: Any) -> AppConfig:\n with belief_scope(\"ConfigManager.save_config\"):\n if isinstance(config, AppConfig):\n logger.reason(\"Saving typed AppConfig payload\")\n self.config = config\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config)\n return self.config\n\n if isinstance(config, dict):\n logger.reason(\n \"Saving raw config payload\",\n extra={\"keys\": sorted(config.keys())},\n )\n self.raw_payload = dict(config)\n typed_config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self.config = typed_config\n self._save_config_to_db(typed_config)\n return self.config\n\n logger.explore(\n \"Unsupported config type supplied to save_config\",\n extra={\"type\": type(config).__name__},\n )\n raise TypeError(\"config must be AppConfig or dict\")\n\n # [/DEF:save_config:Function]\n\n # [DEF:update_global_settings:Function]\n # @PURPOSE: Replace global settings and persist the resulting configuration.\n def update_global_settings(self, settings: GlobalSettings) -> AppConfig:\n with belief_scope(\"ConfigManager.update_global_settings\"):\n logger.reason(\"Updating global settings\")\n self.config.settings = settings\n self.save()\n return self.config\n\n # [/DEF:update_global_settings:Function]\n\n # [DEF:validate_path:Function]\n # @PURPOSE: Validate that path exists and is writable, creating it when absent.\n def validate_path(self, path: str) -> tuple[bool, str]:\n with belief_scope(\"ConfigManager.validate_path\", f\"path={path}\"):\n try:\n target = Path(path).expanduser()\n target.mkdir(parents=True, exist_ok=True)\n\n if not target.exists():\n return False, f\"Path does not exist: {target}\"\n\n if not target.is_dir():\n return False, f\"Path is not a directory: {target}\"\n\n test_file = target / \".write_test\"\n with test_file.open(\"w\", encoding=\"utf-8\") as fh:\n fh.write(\"ok\")\n test_file.unlink(missing_ok=True)\n\n logger.reason(\"Path validation succeeded\", extra={\"path\": str(target)})\n return True, \"OK\"\n except Exception as exc:\n logger.explore(\n \"Path validation failed\", extra={\"path\": path, \"error\": str(exc)}\n )\n return False, str(exc)\n\n # [/DEF:validate_path:Function]\n\n # [DEF:get_environments:Function]\n # @PURPOSE: Return all configured environments.\n def get_environments(self) -> List[Environment]:\n with belief_scope(\"ConfigManager.get_environments\"):\n return list(self.config.environments)\n\n # [/DEF:get_environments:Function]\n\n # [DEF:has_environments:Function]\n # @PURPOSE: Check whether at least one environment exists in configuration.\n def has_environments(self) -> bool:\n with belief_scope(\"ConfigManager.has_environments\"):\n return len(self.config.environments) > 0\n\n # [/DEF:has_environments:Function]\n\n # [DEF:get_environment:Function]\n # @PURPOSE: Resolve a configured environment by identifier.\n def get_environment(self, env_id: str) -> Optional[Environment]:\n with belief_scope(\"ConfigManager.get_environment\", f\"env_id={env_id}\"):\n normalized = str(env_id or \"\").strip()\n if not normalized:\n return None\n\n for env in self.config.environments:\n if env.id == normalized or env.name == normalized:\n return env\n return None\n\n # [/DEF:get_environment:Function]\n\n # [DEF:add_environment:Function]\n # @PURPOSE: Upsert environment by id into configuration and persist.\n def add_environment(self, env: Environment) -> AppConfig:\n with belief_scope(\"ConfigManager.add_environment\", f\"env_id={env.id}\"):\n existing_index = next(\n (\n i\n for i, item in enumerate(self.config.environments)\n if item.id == env.id\n ),\n None,\n )\n if env.is_default:\n for item in self.config.environments:\n item.is_default = False\n\n if existing_index is None:\n logger.reason(\"Appending new environment\", extra={\"env_id\": env.id})\n self.config.environments.append(env)\n else:\n logger.reason(\n \"Replacing existing environment during add\",\n extra={\"env_id\": env.id},\n )\n self.config.environments[existing_index] = env\n\n if len(self.config.environments) == 1 and not any(\n item.is_default for item in self.config.environments\n ):\n self.config.environments[0].is_default = True\n\n self.save()\n return self.config\n\n # [/DEF:add_environment:Function]\n\n # [DEF:update_environment:Function]\n # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.\n def update_environment(self, env_id: str, env: Environment) -> bool:\n with belief_scope(\"ConfigManager.update_environment\", f\"env_id={env_id}\"):\n for index, existing in enumerate(self.config.environments):\n if existing.id != env_id:\n continue\n\n update_data = env.model_dump()\n if update_data.get(\"password\") == \"********\":\n update_data[\"password\"] = existing.password\n\n updated = Environment.model_validate(update_data)\n\n if updated.is_default:\n for item in self.config.environments:\n item.is_default = False\n elif existing.is_default and not updated.is_default:\n updated.is_default = True\n\n self.config.environments[index] = updated\n logger.reason(\"Environment updated\", extra={\"env_id\": env_id})\n self.save()\n return True\n\n logger.explore(\n \"Environment update skipped; env not found\", extra={\"env_id\": env_id}\n )\n return False\n\n # [/DEF:update_environment:Function]\n\n # [DEF:delete_environment:Function]\n # @PURPOSE: Delete environment by id and persist when deletion occurs.\n def delete_environment(self, env_id: str) -> bool:\n with belief_scope(\"ConfigManager.delete_environment\", f\"env_id={env_id}\"):\n before = len(self.config.environments)\n removed = [env for env in self.config.environments if env.id == env_id]\n self.config.environments = [\n env for env in self.config.environments if env.id != env_id\n ]\n\n if len(self.config.environments) == before:\n logger.explore(\n \"Environment delete skipped; env not found\",\n extra={\"env_id\": env_id},\n )\n return False\n\n if removed and removed[0].is_default and self.config.environments:\n self.config.environments[0].is_default = True\n\n if self.config.settings.default_environment_id == env_id:\n replacement = next(\n (env.id for env in self.config.environments if env.is_default), None\n )\n self.config.settings.default_environment_id = replacement\n\n logger.reason(\n \"Environment deleted\",\n extra={\"env_id\": env_id, \"remaining\": len(self.config.environments)},\n )\n self.save()\n return True\n\n # [/DEF:delete_environment:Function]\n\n\n# [/DEF:ConfigManager:Class]\n# [/DEF:ConfigManager:Module]\n" + "body": "# [DEF:ConfigManager:Module]\n#\n# @COMPLEXITY: 5\n# @SEMANTICS: config, manager, persistence, migration, postgresql\n# @PURPOSE: Manages application configuration persistence in DB with one-time migration from legacy JSON.\n# @LAYER: Domain\n# @PRE: Database schema for AppConfigRecord must be initialized.\n# @POST: Configuration is loaded into memory and logger is configured.\n# @SIDE_EFFECT: Performs DB I/O and may update global logging level.\n# @DATA_CONTRACT: Input[json, record] -> Model[AppConfig]\n# @INVARIANT: Configuration must always be representable by AppConfig and persisted under global record id.\n# @RELATION: [DEPENDS_ON] ->[AppConfig]\n# @RELATION: [DEPENDS_ON] ->[SessionLocal]\n# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]\n# @RELATION: [CALLS] ->[logger]\n# @RELATION: [CALLS] ->[configure_logger]\n#\nimport json\nimport os\nfrom pathlib import Path\nfrom typing import Any, Optional, List\n\nfrom sqlalchemy.orm import Session\n\nfrom .config_models import AppConfig, Environment, GlobalSettings\nfrom .database import SessionLocal\nfrom ..models.config import AppConfigRecord\nfrom ..models.mapping import Environment as EnvironmentRecord\nfrom .logger import configure_logger, belief_scope\nfrom .cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"ConfigManager\")\n\n# [DEF:ConfigManager:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Handles application configuration load, validation, mutation, and persistence lifecycle.\n# @PRE: Database is accessible and AppConfigRecord schema is loaded.\n# @POST: Configuration state is synchronized between memory and database.\n# @SIDE_EFFECT: Performs DB I/O, OS path validation, and logger reconfiguration.\nclass ConfigManager:\n # [DEF:__init__:Function]\n # @PURPOSE: Initialize manager state from persisted or migrated configuration.\n # @PRE: config_path is a non-empty string path.\n # @POST: self.config is initialized as AppConfig and logger is configured.\n # @SIDE_EFFECT: Reads config sources and updates logging configuration.\n # @DATA_CONTRACT: Input(str config_path) -> Output(None; self.config: AppConfig)\n def __init__(self, config_path: str = \"config.json\"):\n with belief_scope(\"ConfigManager.__init__\"):\n if not isinstance(config_path, str) or not config_path:\n log.explore(\"Invalid config_path provided\", error=\"config_path must be a non-empty string\", payload={\"path\": config_path})\n raise ValueError(\"config_path must be a non-empty string\")\n\n log.reason(f\"Initializing ConfigManager with legacy path: {config_path}\")\n\n self.config_path = Path(config_path)\n self.raw_payload: dict[str, Any] = {}\n self.config: AppConfig = self._load_config()\n\n configure_logger(self.config.settings.logging)\n\n if not isinstance(self.config, AppConfig):\n log.explore(\"Config loading resulted in invalid type\", error=\"Loaded config is not an AppConfig instance\", payload={\"type\": type(self.config)})\n raise TypeError(\"self.config must be an instance of AppConfig\")\n\n log.reflect(\"ConfigManager initialization complete\")\n\n # [/DEF:__init__:Function]\n\n # [DEF:_apply_features_from_env:Function]\n # @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config.\n # @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place.\n # @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth.\n @staticmethod\n def _apply_features_from_env(settings: GlobalSettings) -> None:\n with belief_scope(\"ConfigManager._apply_features_from_env\"):\n dataset_review_env = os.getenv(\"FEATURES__DATASET_REVIEW\")\n if dataset_review_env is not None:\n parsed = dataset_review_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.dataset_review = parsed\n log.reason(\n \"Applied FEATURES__DATASET_REVIEW from env\",\n payload={\"value\": parsed, \"raw\": dataset_review_env},\n )\n\n health_monitor_env = os.getenv(\"FEATURES__HEALTH_MONITOR\")\n if health_monitor_env is not None:\n parsed = health_monitor_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.health_monitor = parsed\n log.reason(\n \"Applied FEATURES__HEALTH_MONITOR from env\",\n payload={\"value\": parsed, \"raw\": health_monitor_env},\n )\n\n # [/DEF:_apply_features_from_env:Function]\n\n # [DEF:_default_config:Function]\n # @PURPOSE: Build default application configuration fallback.\n def _default_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._default_config\"):\n log.reason(\"Building default AppConfig fallback\")\n config = AppConfig(environments=[], settings=GlobalSettings())\n self._apply_features_from_env(config.settings)\n return config\n\n # [/DEF:_default_config:Function]\n\n # [DEF:_sync_raw_payload_from_config:Function]\n # @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.\n def _sync_raw_payload_from_config(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._sync_raw_payload_from_config\"):\n typed_payload = self.config.model_dump()\n merged_payload = dict(self.raw_payload or {})\n merged_payload[\"environments\"] = typed_payload.get(\"environments\", [])\n merged_payload[\"settings\"] = typed_payload.get(\"settings\", {})\n self.raw_payload = merged_payload\n log.reason(\n \"Synchronized raw payload from typed config\",\n payload={\n \"environments_count\": len(\n merged_payload.get(\"environments\", []) or []\n ),\n \"has_settings\": \"settings\" in merged_payload,\n \"extra_sections\": sorted(\n key\n for key in merged_payload.keys()\n if key not in {\"environments\", \"settings\"}\n ),\n },\n )\n return merged_payload\n\n # [/DEF:_sync_raw_payload_from_config:Function]\n\n # [DEF:_load_from_legacy_file:Function]\n # @PURPOSE: Load legacy JSON configuration for migration fallback path.\n def _load_from_legacy_file(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._load_from_legacy_file\"):\n if not self.config_path.exists():\n log.reason(\n \"Legacy config file not found; using default payload\",\n payload={\"path\": str(self.config_path)},\n )\n return {}\n\n log.reason(\n \"Loading legacy config file\", payload={\"path\": str(self.config_path)}\n )\n with self.config_path.open(\"r\", encoding=\"utf-8\") as fh:\n payload = json.load(fh)\n\n if not isinstance(payload, dict):\n log.explore(\"Legacy config payload is not a JSON object\", error=f\"Expected dict, got {type(payload).__name__}\", payload={\n \"path\": str(self.config_path),\n \"type\": type(payload).__name__,\n })\n raise ValueError(\"Legacy config payload must be a JSON object\")\n\n log.reason(\n \"Legacy config file loaded successfully\",\n payload={\"path\": str(self.config_path), \"keys\": sorted(payload.keys())},\n )\n return payload\n\n # [/DEF:_load_from_legacy_file:Function]\n\n # [DEF:_get_record:Function]\n # @PURPOSE: Resolve global configuration record from DB.\n def _get_record(self, session: Session) -> Optional[AppConfigRecord]:\n with belief_scope(\"ConfigManager._get_record\"):\n record = (\n session.query(AppConfigRecord)\n .filter(AppConfigRecord.id == \"global\")\n .first()\n )\n log.reason(\n \"Resolved app config record\", payload={\"exists\": record is not None}\n )\n return record\n\n # [/DEF:_get_record:Function]\n\n # [DEF:_load_config:Function]\n # @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.\n def _load_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._load_config\"):\n session = SessionLocal()\n try:\n record = self._get_record(session)\n if record and isinstance(record.payload, dict):\n log.reason(\n \"Loading configuration from database\",\n payload={\"record_id\": record.id},\n )\n self.raw_payload = dict(record.payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._sync_environment_records(session, config)\n session.commit()\n log.reason(\n \"Database configuration validated successfully\",\n payload={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n return config\n\n log.reason(\n \"Database configuration record missing; attempting legacy file migration\",\n payload={\"legacy_path\": str(self.config_path)},\n )\n legacy_payload = self._load_from_legacy_file()\n\n if legacy_payload:\n self.raw_payload = dict(legacy_payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._apply_features_from_env(config.settings)\n log.reason(\n \"Legacy payload validated; persisting migrated configuration to database\",\n payload={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n self._save_config_to_db(config, session=session)\n return config\n\n log.reason(\n \"No persisted config found; falling back to default configuration\"\n )\n config = self._default_config()\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config, session=session)\n return config\n except (json.JSONDecodeError, TypeError, ValueError) as exc:\n log.explore(\"Recoverable config load failure; falling back to default configuration\", error=str(exc), payload={\"legacy_path\": str(self.config_path)})\n config = self._default_config()\n self.raw_payload = config.model_dump()\n return config\n except Exception as exc:\n log.explore(\"Critical config load failure; re-raising persistence or validation error\", error=str(exc))\n raise\n finally:\n session.close()\n\n # [/DEF:_load_config:Function]\n\n # [DEF:_sync_environment_records:Function]\n # @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models.\n def _sync_environment_records(self, session: Session, config: AppConfig) -> None:\n with belief_scope(\"ConfigManager._sync_environment_records\"):\n configured_envs = list(config.environments or [])\n persisted_records = session.query(EnvironmentRecord).all()\n persisted_by_id = {\n str(record.id or \"\").strip(): record for record in persisted_records\n }\n\n for environment in configured_envs:\n normalized_id = str(environment.id or \"\").strip()\n if not normalized_id:\n continue\n\n display_name = (\n str(environment.name or normalized_id).strip() or normalized_id\n )\n normalized_url = str(environment.url or \"\").strip()\n credentials_id = (\n str(environment.username or \"\").strip() or normalized_id\n )\n\n record = persisted_by_id.get(normalized_id)\n if record is None:\n log.reason(\n \"Creating relational environment record from typed config\",\n payload={\n \"environment_id\": normalized_id,\n \"environment_name\": display_name,\n },\n )\n session.add(\n EnvironmentRecord(\n id=normalized_id,\n name=display_name,\n url=normalized_url,\n credentials_id=credentials_id,\n )\n )\n continue\n\n record.name = display_name\n record.url = normalized_url\n record.credentials_id = credentials_id\n\n # [/DEF:_sync_environment_records:Function]\n\n # [DEF:_save_config_to_db:Function]\n # @PURPOSE: Persist provided AppConfig into the global DB configuration record.\n def _save_config_to_db(\n self, config: AppConfig, session: Optional[Session] = None\n ) -> None:\n with belief_scope(\"ConfigManager._save_config_to_db\"):\n owns_session = session is None\n db = session or SessionLocal()\n try:\n self.config = config\n payload = self._sync_raw_payload_from_config()\n record = self._get_record(db)\n if record is None:\n log.reason(\"Creating new global app config record\")\n record = AppConfigRecord(id=\"global\", payload=payload)\n db.add(record)\n else:\n log.reason(\n \"Updating existing global app config record\",\n payload={\"record_id\": record.id},\n )\n record.payload = payload\n\n self._sync_environment_records(db, config)\n\n db.commit()\n log.reason(\n \"Configuration persisted to database\",\n payload={\n \"environments_count\": len(\n payload.get(\"environments\", []) or []\n ),\n \"payload_keys\": sorted(payload.keys()),\n },\n )\n except Exception:\n db.rollback()\n log.explore(\"Database save failed; transaction rolled back\", error=\"Database transaction rolled back\")\n raise\n finally:\n if owns_session:\n db.close()\n\n # [/DEF:_save_config_to_db:Function]\n\n # [DEF:save:Function]\n # @PURPOSE: Persist current in-memory configuration state.\n def save(self) -> None:\n with belief_scope(\"ConfigManager.save\"):\n log.reason(\"Persisting current in-memory configuration\")\n self._save_config_to_db(self.config)\n\n # [/DEF:save:Function]\n\n # [DEF:get_config:Function]\n # @PURPOSE: Return current in-memory configuration snapshot.\n def get_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager.get_config\"):\n return self.config\n\n # [/DEF:get_config:Function]\n\n # [DEF:get_payload:Function]\n # @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema.\n def get_payload(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager.get_payload\"):\n return self._sync_raw_payload_from_config()\n\n # [/DEF:get_payload:Function]\n\n # [DEF:save_config:Function]\n # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.\n def save_config(self, config: Any) -> AppConfig:\n with belief_scope(\"ConfigManager.save_config\"):\n if isinstance(config, AppConfig):\n log.reason(\"Saving typed AppConfig payload\")\n self.config = config\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config)\n return self.config\n\n if isinstance(config, dict):\n log.reason(\n \"Saving raw config payload\",\n payload={\"keys\": sorted(config.keys())},\n )\n self.raw_payload = dict(config)\n typed_config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self.config = typed_config\n self._save_config_to_db(typed_config)\n return self.config\n\n log.explore(\"Unsupported config type supplied to save_config\", error=f\"Expected AppConfig or dict, got {type(config).__name__}\", payload={\"type\": type(config).__name__})\n raise TypeError(\"config must be AppConfig or dict\")\n\n # [/DEF:save_config:Function]\n\n # [DEF:update_global_settings:Function]\n # @PURPOSE: Replace global settings and persist the resulting configuration.\n def update_global_settings(self, settings: GlobalSettings) -> AppConfig:\n with belief_scope(\"ConfigManager.update_global_settings\"):\n log.reason(\"Updating global settings\")\n self.config.settings = settings\n self.save()\n return self.config\n\n # [/DEF:update_global_settings:Function]\n\n # [DEF:validate_path:Function]\n # @PURPOSE: Validate that path exists and is writable, creating it when absent.\n def validate_path(self, path: str) -> tuple[bool, str]:\n with belief_scope(\"ConfigManager.validate_path\", f\"path={path}\"):\n try:\n target = Path(path).expanduser()\n target.mkdir(parents=True, exist_ok=True)\n\n if not target.exists():\n return False, f\"Path does not exist: {target}\"\n\n if not target.is_dir():\n return False, f\"Path is not a directory: {target}\"\n\n test_file = target / \".write_test\"\n with test_file.open(\"w\", encoding=\"utf-8\") as fh:\n fh.write(\"ok\")\n test_file.unlink(missing_ok=True)\n\n log.reason(\"Path validation succeeded\", payload={\"path\": str(target)})\n return True, \"OK\"\n except Exception as exc:\n log.explore(\"Path validation failed\", error=str(exc), payload={\"path\": path})\n return False, str(exc)\n\n # [/DEF:validate_path:Function]\n\n # [DEF:get_environments:Function]\n # @PURPOSE: Return all configured environments.\n def get_environments(self) -> List[Environment]:\n with belief_scope(\"ConfigManager.get_environments\"):\n return list(self.config.environments)\n\n # [/DEF:get_environments:Function]\n\n # [DEF:has_environments:Function]\n # @PURPOSE: Check whether at least one environment exists in configuration.\n def has_environments(self) -> bool:\n with belief_scope(\"ConfigManager.has_environments\"):\n return len(self.config.environments) > 0\n\n # [/DEF:has_environments:Function]\n\n # [DEF:get_environment:Function]\n # @PURPOSE: Resolve a configured environment by identifier.\n def get_environment(self, env_id: str) -> Optional[Environment]:\n with belief_scope(\"ConfigManager.get_environment\", f\"env_id={env_id}\"):\n normalized = str(env_id or \"\").strip()\n if not normalized:\n return None\n\n for env in self.config.environments:\n if env.id == normalized or env.name == normalized:\n return env\n return None\n\n # [/DEF:get_environment:Function]\n\n # [DEF:add_environment:Function]\n # @PURPOSE: Upsert environment by id into configuration and persist.\n def add_environment(self, env: Environment) -> AppConfig:\n with belief_scope(\"ConfigManager.add_environment\", f\"env_id={env.id}\"):\n existing_index = next(\n (\n i\n for i, item in enumerate(self.config.environments)\n if item.id == env.id\n ),\n None,\n )\n if env.is_default:\n for item in self.config.environments:\n item.is_default = False\n\n if existing_index is None:\n log.reason(\"Appending new environment\", payload={\"env_id\": env.id})\n self.config.environments.append(env)\n else:\n log.reason(\n \"Replacing existing environment during add\",\n payload={\"env_id\": env.id},\n )\n self.config.environments[existing_index] = env\n\n if len(self.config.environments) == 1 and not any(\n item.is_default for item in self.config.environments\n ):\n self.config.environments[0].is_default = True\n\n self.save()\n return self.config\n\n # [/DEF:add_environment:Function]\n\n # [DEF:update_environment:Function]\n # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.\n def update_environment(self, env_id: str, env: Environment) -> bool:\n with belief_scope(\"ConfigManager.update_environment\", f\"env_id={env_id}\"):\n for index, existing in enumerate(self.config.environments):\n if existing.id != env_id:\n continue\n\n update_data = env.model_dump()\n if update_data.get(\"password\") == \"********\":\n update_data[\"password\"] = existing.password\n\n updated = Environment.model_validate(update_data)\n\n if updated.is_default:\n for item in self.config.environments:\n item.is_default = False\n elif existing.is_default and not updated.is_default:\n updated.is_default = True\n\n self.config.environments[index] = updated\n log.reason(\"Environment updated\", payload={\"env_id\": env_id})\n self.save()\n return True\n\n log.explore(\"Environment update skipped; env not found\", error=f\"No matching environment found for id: {env_id}\", payload={\"env_id\": env_id})\n return False\n\n # [/DEF:update_environment:Function]\n\n # [DEF:delete_environment:Function]\n # @PURPOSE: Delete environment by id and persist when deletion occurs.\n def delete_environment(self, env_id: str) -> bool:\n with belief_scope(\"ConfigManager.delete_environment\", f\"env_id={env_id}\"):\n before = len(self.config.environments)\n removed = [env for env in self.config.environments if env.id == env_id]\n self.config.environments = [\n env for env in self.config.environments if env.id != env_id\n ]\n\n if len(self.config.environments) == before:\n log.explore(\"Environment delete skipped; env not found\", error=f\"No matching environment found for id: {env_id}\", payload={\"env_id\": env_id})\n return False\n\n if removed and removed[0].is_default and self.config.environments:\n self.config.environments[0].is_default = True\n\n if self.config.settings.default_environment_id == env_id:\n replacement = next(\n (env.id for env in self.config.environments if env.is_default), None\n )\n self.config.settings.default_environment_id = replacement\n\n log.reason(\n \"Environment deleted\",\n payload={\"env_id\": env_id, \"remaining\": len(self.config.environments)},\n )\n self.save()\n return True\n\n # [/DEF:delete_environment:Function]\n\n\n# [/DEF:ConfigManager:Class]\n# [/DEF:ConfigManager:Module]\n" }, { "contract_id": "_apply_features_from_env", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 72, - "end_line": 97, + "start_line": 69, + "end_line": 94, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31901,14 +31901,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_apply_features_from_env:Function]\n # @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config.\n # @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place.\n # @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth.\n @staticmethod\n def _apply_features_from_env(settings: GlobalSettings) -> None:\n with belief_scope(\"ConfigManager._apply_features_from_env\"):\n dataset_review_env = os.getenv(\"FEATURES__DATASET_REVIEW\")\n if dataset_review_env is not None:\n parsed = dataset_review_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.dataset_review = parsed\n logger.reason(\n \"Applied FEATURES__DATASET_REVIEW from env\",\n extra={\"value\": parsed, \"raw\": dataset_review_env},\n )\n\n health_monitor_env = os.getenv(\"FEATURES__HEALTH_MONITOR\")\n if health_monitor_env is not None:\n parsed = health_monitor_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.health_monitor = parsed\n logger.reason(\n \"Applied FEATURES__HEALTH_MONITOR from env\",\n extra={\"value\": parsed, \"raw\": health_monitor_env},\n )\n\n # [/DEF:_apply_features_from_env:Function]\n" + "body": " # [DEF:_apply_features_from_env:Function]\n # @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config.\n # @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place.\n # @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth.\n @staticmethod\n def _apply_features_from_env(settings: GlobalSettings) -> None:\n with belief_scope(\"ConfigManager._apply_features_from_env\"):\n dataset_review_env = os.getenv(\"FEATURES__DATASET_REVIEW\")\n if dataset_review_env is not None:\n parsed = dataset_review_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.dataset_review = parsed\n log.reason(\n \"Applied FEATURES__DATASET_REVIEW from env\",\n payload={\"value\": parsed, \"raw\": dataset_review_env},\n )\n\n health_monitor_env = os.getenv(\"FEATURES__HEALTH_MONITOR\")\n if health_monitor_env is not None:\n parsed = health_monitor_env.strip().lower() in (\"true\", \"1\", \"yes\")\n settings.features.health_monitor = parsed\n log.reason(\n \"Applied FEATURES__HEALTH_MONITOR from env\",\n payload={\"value\": parsed, \"raw\": health_monitor_env},\n )\n\n # [/DEF:_apply_features_from_env:Function]\n" }, { "contract_id": "_default_config", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 99, - "end_line": 108, + "start_line": 96, + "end_line": 105, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31917,14 +31917,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_default_config:Function]\n # @PURPOSE: Build default application configuration fallback.\n def _default_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._default_config\"):\n logger.reason(\"Building default AppConfig fallback\")\n config = AppConfig(environments=[], settings=GlobalSettings())\n self._apply_features_from_env(config.settings)\n return config\n\n # [/DEF:_default_config:Function]\n" + "body": " # [DEF:_default_config:Function]\n # @PURPOSE: Build default application configuration fallback.\n def _default_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._default_config\"):\n log.reason(\"Building default AppConfig fallback\")\n config = AppConfig(environments=[], settings=GlobalSettings())\n self._apply_features_from_env(config.settings)\n return config\n\n # [/DEF:_default_config:Function]\n" }, { "contract_id": "_sync_raw_payload_from_config", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 110, - "end_line": 135, + "start_line": 107, + "end_line": 132, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31933,14 +31933,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_sync_raw_payload_from_config:Function]\n # @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.\n def _sync_raw_payload_from_config(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._sync_raw_payload_from_config\"):\n typed_payload = self.config.model_dump()\n merged_payload = dict(self.raw_payload or {})\n merged_payload[\"environments\"] = typed_payload.get(\"environments\", [])\n merged_payload[\"settings\"] = typed_payload.get(\"settings\", {})\n self.raw_payload = merged_payload\n logger.reason(\n \"Synchronized raw payload from typed config\",\n extra={\n \"environments_count\": len(\n merged_payload.get(\"environments\", []) or []\n ),\n \"has_settings\": \"settings\" in merged_payload,\n \"extra_sections\": sorted(\n key\n for key in merged_payload.keys()\n if key not in {\"environments\", \"settings\"}\n ),\n },\n )\n return merged_payload\n\n # [/DEF:_sync_raw_payload_from_config:Function]\n" + "body": " # [DEF:_sync_raw_payload_from_config:Function]\n # @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.\n def _sync_raw_payload_from_config(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._sync_raw_payload_from_config\"):\n typed_payload = self.config.model_dump()\n merged_payload = dict(self.raw_payload or {})\n merged_payload[\"environments\"] = typed_payload.get(\"environments\", [])\n merged_payload[\"settings\"] = typed_payload.get(\"settings\", {})\n self.raw_payload = merged_payload\n log.reason(\n \"Synchronized raw payload from typed config\",\n payload={\n \"environments_count\": len(\n merged_payload.get(\"environments\", []) or []\n ),\n \"has_settings\": \"settings\" in merged_payload,\n \"extra_sections\": sorted(\n key\n for key in merged_payload.keys()\n if key not in {\"environments\", \"settings\"}\n ),\n },\n )\n return merged_payload\n\n # [/DEF:_sync_raw_payload_from_config:Function]\n" }, { "contract_id": "_load_from_legacy_file", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 137, - "end_line": 170, + "start_line": 134, + "end_line": 164, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31949,14 +31949,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_load_from_legacy_file:Function]\n # @PURPOSE: Load legacy JSON configuration for migration fallback path.\n def _load_from_legacy_file(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._load_from_legacy_file\"):\n if not self.config_path.exists():\n logger.reason(\n \"Legacy config file not found; using default payload\",\n extra={\"path\": str(self.config_path)},\n )\n return {}\n\n logger.reason(\n \"Loading legacy config file\", extra={\"path\": str(self.config_path)}\n )\n with self.config_path.open(\"r\", encoding=\"utf-8\") as fh:\n payload = json.load(fh)\n\n if not isinstance(payload, dict):\n logger.explore(\n \"Legacy config payload is not a JSON object\",\n extra={\n \"path\": str(self.config_path),\n \"type\": type(payload).__name__,\n },\n )\n raise ValueError(\"Legacy config payload must be a JSON object\")\n\n logger.reason(\n \"Legacy config file loaded successfully\",\n extra={\"path\": str(self.config_path), \"keys\": sorted(payload.keys())},\n )\n return payload\n\n # [/DEF:_load_from_legacy_file:Function]\n" + "body": " # [DEF:_load_from_legacy_file:Function]\n # @PURPOSE: Load legacy JSON configuration for migration fallback path.\n def _load_from_legacy_file(self) -> dict[str, Any]:\n with belief_scope(\"ConfigManager._load_from_legacy_file\"):\n if not self.config_path.exists():\n log.reason(\n \"Legacy config file not found; using default payload\",\n payload={\"path\": str(self.config_path)},\n )\n return {}\n\n log.reason(\n \"Loading legacy config file\", payload={\"path\": str(self.config_path)}\n )\n with self.config_path.open(\"r\", encoding=\"utf-8\") as fh:\n payload = json.load(fh)\n\n if not isinstance(payload, dict):\n log.explore(\"Legacy config payload is not a JSON object\", error=f\"Expected dict, got {type(payload).__name__}\", payload={\n \"path\": str(self.config_path),\n \"type\": type(payload).__name__,\n })\n raise ValueError(\"Legacy config payload must be a JSON object\")\n\n log.reason(\n \"Legacy config file loaded successfully\",\n payload={\"path\": str(self.config_path), \"keys\": sorted(payload.keys())},\n )\n return payload\n\n # [/DEF:_load_from_legacy_file:Function]\n" }, { "contract_id": "_get_record", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 172, - "end_line": 186, + "start_line": 166, + "end_line": 180, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31965,14 +31965,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_get_record:Function]\n # @PURPOSE: Resolve global configuration record from DB.\n def _get_record(self, session: Session) -> Optional[AppConfigRecord]:\n with belief_scope(\"ConfigManager._get_record\"):\n record = (\n session.query(AppConfigRecord)\n .filter(AppConfigRecord.id == \"global\")\n .first()\n )\n logger.reason(\n \"Resolved app config record\", extra={\"exists\": record is not None}\n )\n return record\n\n # [/DEF:_get_record:Function]\n" + "body": " # [DEF:_get_record:Function]\n # @PURPOSE: Resolve global configuration record from DB.\n def _get_record(self, session: Session) -> Optional[AppConfigRecord]:\n with belief_scope(\"ConfigManager._get_record\"):\n record = (\n session.query(AppConfigRecord)\n .filter(AppConfigRecord.id == \"global\")\n .first()\n )\n log.reason(\n \"Resolved app config record\", payload={\"exists\": record is not None}\n )\n return record\n\n # [/DEF:_get_record:Function]\n" }, { "contract_id": "_load_config", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 188, - "end_line": 267, + "start_line": 182, + "end_line": 255, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31981,14 +31981,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_load_config:Function]\n # @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.\n def _load_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._load_config\"):\n session = SessionLocal()\n try:\n record = self._get_record(session)\n if record and isinstance(record.payload, dict):\n logger.reason(\n \"Loading configuration from database\",\n extra={\"record_id\": record.id},\n )\n self.raw_payload = dict(record.payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._sync_environment_records(session, config)\n session.commit()\n logger.reason(\n \"Database configuration validated successfully\",\n extra={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n return config\n\n logger.reason(\n \"Database configuration record missing; attempting legacy file migration\",\n extra={\"legacy_path\": str(self.config_path)},\n )\n legacy_payload = self._load_from_legacy_file()\n\n if legacy_payload:\n self.raw_payload = dict(legacy_payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._apply_features_from_env(config.settings)\n logger.reason(\n \"Legacy payload validated; persisting migrated configuration to database\",\n extra={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n self._save_config_to_db(config, session=session)\n return config\n\n logger.reason(\n \"No persisted config found; falling back to default configuration\"\n )\n config = self._default_config()\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config, session=session)\n return config\n except (json.JSONDecodeError, TypeError, ValueError) as exc:\n logger.explore(\n \"Recoverable config load failure; falling back to default configuration\",\n extra={\"error\": str(exc), \"legacy_path\": str(self.config_path)},\n )\n config = self._default_config()\n self.raw_payload = config.model_dump()\n return config\n except Exception as exc:\n logger.explore(\n \"Critical config load failure; re-raising persistence or validation error\",\n extra={\"error\": str(exc)},\n )\n raise\n finally:\n session.close()\n\n # [/DEF:_load_config:Function]\n" + "body": " # [DEF:_load_config:Function]\n # @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.\n def _load_config(self) -> AppConfig:\n with belief_scope(\"ConfigManager._load_config\"):\n session = SessionLocal()\n try:\n record = self._get_record(session)\n if record and isinstance(record.payload, dict):\n log.reason(\n \"Loading configuration from database\",\n payload={\"record_id\": record.id},\n )\n self.raw_payload = dict(record.payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._sync_environment_records(session, config)\n session.commit()\n log.reason(\n \"Database configuration validated successfully\",\n payload={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n return config\n\n log.reason(\n \"Database configuration record missing; attempting legacy file migration\",\n payload={\"legacy_path\": str(self.config_path)},\n )\n legacy_payload = self._load_from_legacy_file()\n\n if legacy_payload:\n self.raw_payload = dict(legacy_payload)\n config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self._apply_features_from_env(config.settings)\n log.reason(\n \"Legacy payload validated; persisting migrated configuration to database\",\n payload={\n \"environments_count\": len(config.environments),\n \"payload_keys\": sorted(self.raw_payload.keys()),\n },\n )\n self._save_config_to_db(config, session=session)\n return config\n\n log.reason(\n \"No persisted config found; falling back to default configuration\"\n )\n config = self._default_config()\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config, session=session)\n return config\n except (json.JSONDecodeError, TypeError, ValueError) as exc:\n log.explore(\"Recoverable config load failure; falling back to default configuration\", error=str(exc), payload={\"legacy_path\": str(self.config_path)})\n config = self._default_config()\n self.raw_payload = config.model_dump()\n return config\n except Exception as exc:\n log.explore(\"Critical config load failure; re-raising persistence or validation error\", error=str(exc))\n raise\n finally:\n session.close()\n\n # [/DEF:_load_config:Function]\n" }, { "contract_id": "_sync_environment_records", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 269, - "end_line": 315, + "start_line": 257, + "end_line": 303, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -31997,14 +31997,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_sync_environment_records:Function]\n # @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models.\n def _sync_environment_records(self, session: Session, config: AppConfig) -> None:\n with belief_scope(\"ConfigManager._sync_environment_records\"):\n configured_envs = list(config.environments or [])\n persisted_records = session.query(EnvironmentRecord).all()\n persisted_by_id = {\n str(record.id or \"\").strip(): record for record in persisted_records\n }\n\n for environment in configured_envs:\n normalized_id = str(environment.id or \"\").strip()\n if not normalized_id:\n continue\n\n display_name = (\n str(environment.name or normalized_id).strip() or normalized_id\n )\n normalized_url = str(environment.url or \"\").strip()\n credentials_id = (\n str(environment.username or \"\").strip() or normalized_id\n )\n\n record = persisted_by_id.get(normalized_id)\n if record is None:\n logger.reason(\n \"Creating relational environment record from typed config\",\n extra={\n \"environment_id\": normalized_id,\n \"environment_name\": display_name,\n },\n )\n session.add(\n EnvironmentRecord(\n id=normalized_id,\n name=display_name,\n url=normalized_url,\n credentials_id=credentials_id,\n )\n )\n continue\n\n record.name = display_name\n record.url = normalized_url\n record.credentials_id = credentials_id\n\n # [/DEF:_sync_environment_records:Function]\n" + "body": " # [DEF:_sync_environment_records:Function]\n # @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models.\n def _sync_environment_records(self, session: Session, config: AppConfig) -> None:\n with belief_scope(\"ConfigManager._sync_environment_records\"):\n configured_envs = list(config.environments or [])\n persisted_records = session.query(EnvironmentRecord).all()\n persisted_by_id = {\n str(record.id or \"\").strip(): record for record in persisted_records\n }\n\n for environment in configured_envs:\n normalized_id = str(environment.id or \"\").strip()\n if not normalized_id:\n continue\n\n display_name = (\n str(environment.name or normalized_id).strip() or normalized_id\n )\n normalized_url = str(environment.url or \"\").strip()\n credentials_id = (\n str(environment.username or \"\").strip() or normalized_id\n )\n\n record = persisted_by_id.get(normalized_id)\n if record is None:\n log.reason(\n \"Creating relational environment record from typed config\",\n payload={\n \"environment_id\": normalized_id,\n \"environment_name\": display_name,\n },\n )\n session.add(\n EnvironmentRecord(\n id=normalized_id,\n name=display_name,\n url=normalized_url,\n credentials_id=credentials_id,\n )\n )\n continue\n\n record.name = display_name\n record.url = normalized_url\n record.credentials_id = credentials_id\n\n # [/DEF:_sync_environment_records:Function]\n" }, { "contract_id": "_save_config_to_db", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 317, - "end_line": 360, + "start_line": 305, + "end_line": 348, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32013,14 +32013,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:_save_config_to_db:Function]\n # @PURPOSE: Persist provided AppConfig into the global DB configuration record.\n def _save_config_to_db(\n self, config: AppConfig, session: Optional[Session] = None\n ) -> None:\n with belief_scope(\"ConfigManager._save_config_to_db\"):\n owns_session = session is None\n db = session or SessionLocal()\n try:\n self.config = config\n payload = self._sync_raw_payload_from_config()\n record = self._get_record(db)\n if record is None:\n logger.reason(\"Creating new global app config record\")\n record = AppConfigRecord(id=\"global\", payload=payload)\n db.add(record)\n else:\n logger.reason(\n \"Updating existing global app config record\",\n extra={\"record_id\": record.id},\n )\n record.payload = payload\n\n self._sync_environment_records(db, config)\n\n db.commit()\n logger.reason(\n \"Configuration persisted to database\",\n extra={\n \"environments_count\": len(\n payload.get(\"environments\", []) or []\n ),\n \"payload_keys\": sorted(payload.keys()),\n },\n )\n except Exception:\n db.rollback()\n logger.explore(\"Database save failed; transaction rolled back\")\n raise\n finally:\n if owns_session:\n db.close()\n\n # [/DEF:_save_config_to_db:Function]\n" + "body": " # [DEF:_save_config_to_db:Function]\n # @PURPOSE: Persist provided AppConfig into the global DB configuration record.\n def _save_config_to_db(\n self, config: AppConfig, session: Optional[Session] = None\n ) -> None:\n with belief_scope(\"ConfigManager._save_config_to_db\"):\n owns_session = session is None\n db = session or SessionLocal()\n try:\n self.config = config\n payload = self._sync_raw_payload_from_config()\n record = self._get_record(db)\n if record is None:\n log.reason(\"Creating new global app config record\")\n record = AppConfigRecord(id=\"global\", payload=payload)\n db.add(record)\n else:\n log.reason(\n \"Updating existing global app config record\",\n payload={\"record_id\": record.id},\n )\n record.payload = payload\n\n self._sync_environment_records(db, config)\n\n db.commit()\n log.reason(\n \"Configuration persisted to database\",\n payload={\n \"environments_count\": len(\n payload.get(\"environments\", []) or []\n ),\n \"payload_keys\": sorted(payload.keys()),\n },\n )\n except Exception:\n db.rollback()\n log.explore(\"Database save failed; transaction rolled back\", error=\"Database transaction rolled back\")\n raise\n finally:\n if owns_session:\n db.close()\n\n # [/DEF:_save_config_to_db:Function]\n" }, { "contract_id": "save", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 362, - "end_line": 369, + "start_line": 350, + "end_line": 357, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32029,14 +32029,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:save:Function]\n # @PURPOSE: Persist current in-memory configuration state.\n def save(self) -> None:\n with belief_scope(\"ConfigManager.save\"):\n logger.reason(\"Persisting current in-memory configuration\")\n self._save_config_to_db(self.config)\n\n # [/DEF:save:Function]\n" + "body": " # [DEF:save:Function]\n # @PURPOSE: Persist current in-memory configuration state.\n def save(self) -> None:\n with belief_scope(\"ConfigManager.save\"):\n log.reason(\"Persisting current in-memory configuration\")\n self._save_config_to_db(self.config)\n\n # [/DEF:save:Function]\n" }, { "contract_id": "get_config", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 371, - "end_line": 377, + "start_line": 359, + "end_line": 365, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32051,8 +32051,8 @@ "contract_id": "get_payload", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 379, - "end_line": 385, + "start_line": 367, + "end_line": 373, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32067,8 +32067,8 @@ "contract_id": "save_config", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 387, - "end_line": 420, + "start_line": 375, + "end_line": 405, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32077,14 +32077,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:save_config:Function]\n # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.\n def save_config(self, config: Any) -> AppConfig:\n with belief_scope(\"ConfigManager.save_config\"):\n if isinstance(config, AppConfig):\n logger.reason(\"Saving typed AppConfig payload\")\n self.config = config\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config)\n return self.config\n\n if isinstance(config, dict):\n logger.reason(\n \"Saving raw config payload\",\n extra={\"keys\": sorted(config.keys())},\n )\n self.raw_payload = dict(config)\n typed_config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self.config = typed_config\n self._save_config_to_db(typed_config)\n return self.config\n\n logger.explore(\n \"Unsupported config type supplied to save_config\",\n extra={\"type\": type(config).__name__},\n )\n raise TypeError(\"config must be AppConfig or dict\")\n\n # [/DEF:save_config:Function]\n" + "body": " # [DEF:save_config:Function]\n # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.\n def save_config(self, config: Any) -> AppConfig:\n with belief_scope(\"ConfigManager.save_config\"):\n if isinstance(config, AppConfig):\n log.reason(\"Saving typed AppConfig payload\")\n self.config = config\n self.raw_payload = config.model_dump()\n self._save_config_to_db(config)\n return self.config\n\n if isinstance(config, dict):\n log.reason(\n \"Saving raw config payload\",\n payload={\"keys\": sorted(config.keys())},\n )\n self.raw_payload = dict(config)\n typed_config = AppConfig.model_validate(\n {\n \"environments\": self.raw_payload.get(\"environments\", []),\n \"settings\": self.raw_payload.get(\"settings\", {}),\n }\n )\n self.config = typed_config\n self._save_config_to_db(typed_config)\n return self.config\n\n log.explore(\"Unsupported config type supplied to save_config\", error=f\"Expected AppConfig or dict, got {type(config).__name__}\", payload={\"type\": type(config).__name__})\n raise TypeError(\"config must be AppConfig or dict\")\n\n # [/DEF:save_config:Function]\n" }, { "contract_id": "update_global_settings", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 422, - "end_line": 431, + "start_line": 407, + "end_line": 416, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32093,14 +32093,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:update_global_settings:Function]\n # @PURPOSE: Replace global settings and persist the resulting configuration.\n def update_global_settings(self, settings: GlobalSettings) -> AppConfig:\n with belief_scope(\"ConfigManager.update_global_settings\"):\n logger.reason(\"Updating global settings\")\n self.config.settings = settings\n self.save()\n return self.config\n\n # [/DEF:update_global_settings:Function]\n" + "body": " # [DEF:update_global_settings:Function]\n # @PURPOSE: Replace global settings and persist the resulting configuration.\n def update_global_settings(self, settings: GlobalSettings) -> AppConfig:\n with belief_scope(\"ConfigManager.update_global_settings\"):\n log.reason(\"Updating global settings\")\n self.config.settings = settings\n self.save()\n return self.config\n\n # [/DEF:update_global_settings:Function]\n" }, { "contract_id": "validate_path", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 433, - "end_line": 460, + "start_line": 418, + "end_line": 443, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32109,14 +32109,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:validate_path:Function]\n # @PURPOSE: Validate that path exists and is writable, creating it when absent.\n def validate_path(self, path: str) -> tuple[bool, str]:\n with belief_scope(\"ConfigManager.validate_path\", f\"path={path}\"):\n try:\n target = Path(path).expanduser()\n target.mkdir(parents=True, exist_ok=True)\n\n if not target.exists():\n return False, f\"Path does not exist: {target}\"\n\n if not target.is_dir():\n return False, f\"Path is not a directory: {target}\"\n\n test_file = target / \".write_test\"\n with test_file.open(\"w\", encoding=\"utf-8\") as fh:\n fh.write(\"ok\")\n test_file.unlink(missing_ok=True)\n\n logger.reason(\"Path validation succeeded\", extra={\"path\": str(target)})\n return True, \"OK\"\n except Exception as exc:\n logger.explore(\n \"Path validation failed\", extra={\"path\": path, \"error\": str(exc)}\n )\n return False, str(exc)\n\n # [/DEF:validate_path:Function]\n" + "body": " # [DEF:validate_path:Function]\n # @PURPOSE: Validate that path exists and is writable, creating it when absent.\n def validate_path(self, path: str) -> tuple[bool, str]:\n with belief_scope(\"ConfigManager.validate_path\", f\"path={path}\"):\n try:\n target = Path(path).expanduser()\n target.mkdir(parents=True, exist_ok=True)\n\n if not target.exists():\n return False, f\"Path does not exist: {target}\"\n\n if not target.is_dir():\n return False, f\"Path is not a directory: {target}\"\n\n test_file = target / \".write_test\"\n with test_file.open(\"w\", encoding=\"utf-8\") as fh:\n fh.write(\"ok\")\n test_file.unlink(missing_ok=True)\n\n log.reason(\"Path validation succeeded\", payload={\"path\": str(target)})\n return True, \"OK\"\n except Exception as exc:\n log.explore(\"Path validation failed\", error=str(exc), payload={\"path\": path})\n return False, str(exc)\n\n # [/DEF:validate_path:Function]\n" }, { "contract_id": "get_environments", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 462, - "end_line": 468, + "start_line": 445, + "end_line": 451, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32131,8 +32131,8 @@ "contract_id": "has_environments", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 470, - "end_line": 476, + "start_line": 453, + "end_line": 459, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32147,8 +32147,8 @@ "contract_id": "get_environment", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 478, - "end_line": 491, + "start_line": 461, + "end_line": 474, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32163,8 +32163,8 @@ "contract_id": "add_environment", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 493, - "end_line": 527, + "start_line": 476, + "end_line": 510, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32173,14 +32173,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:add_environment:Function]\n # @PURPOSE: Upsert environment by id into configuration and persist.\n def add_environment(self, env: Environment) -> AppConfig:\n with belief_scope(\"ConfigManager.add_environment\", f\"env_id={env.id}\"):\n existing_index = next(\n (\n i\n for i, item in enumerate(self.config.environments)\n if item.id == env.id\n ),\n None,\n )\n if env.is_default:\n for item in self.config.environments:\n item.is_default = False\n\n if existing_index is None:\n logger.reason(\"Appending new environment\", extra={\"env_id\": env.id})\n self.config.environments.append(env)\n else:\n logger.reason(\n \"Replacing existing environment during add\",\n extra={\"env_id\": env.id},\n )\n self.config.environments[existing_index] = env\n\n if len(self.config.environments) == 1 and not any(\n item.is_default for item in self.config.environments\n ):\n self.config.environments[0].is_default = True\n\n self.save()\n return self.config\n\n # [/DEF:add_environment:Function]\n" + "body": " # [DEF:add_environment:Function]\n # @PURPOSE: Upsert environment by id into configuration and persist.\n def add_environment(self, env: Environment) -> AppConfig:\n with belief_scope(\"ConfigManager.add_environment\", f\"env_id={env.id}\"):\n existing_index = next(\n (\n i\n for i, item in enumerate(self.config.environments)\n if item.id == env.id\n ),\n None,\n )\n if env.is_default:\n for item in self.config.environments:\n item.is_default = False\n\n if existing_index is None:\n log.reason(\"Appending new environment\", payload={\"env_id\": env.id})\n self.config.environments.append(env)\n else:\n log.reason(\n \"Replacing existing environment during add\",\n payload={\"env_id\": env.id},\n )\n self.config.environments[existing_index] = env\n\n if len(self.config.environments) == 1 and not any(\n item.is_default for item in self.config.environments\n ):\n self.config.environments[0].is_default = True\n\n self.save()\n return self.config\n\n # [/DEF:add_environment:Function]\n" }, { "contract_id": "update_environment", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 529, - "end_line": 559, + "start_line": 512, + "end_line": 540, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32189,14 +32189,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:update_environment:Function]\n # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.\n def update_environment(self, env_id: str, env: Environment) -> bool:\n with belief_scope(\"ConfigManager.update_environment\", f\"env_id={env_id}\"):\n for index, existing in enumerate(self.config.environments):\n if existing.id != env_id:\n continue\n\n update_data = env.model_dump()\n if update_data.get(\"password\") == \"********\":\n update_data[\"password\"] = existing.password\n\n updated = Environment.model_validate(update_data)\n\n if updated.is_default:\n for item in self.config.environments:\n item.is_default = False\n elif existing.is_default and not updated.is_default:\n updated.is_default = True\n\n self.config.environments[index] = updated\n logger.reason(\"Environment updated\", extra={\"env_id\": env_id})\n self.save()\n return True\n\n logger.explore(\n \"Environment update skipped; env not found\", extra={\"env_id\": env_id}\n )\n return False\n\n # [/DEF:update_environment:Function]\n" + "body": " # [DEF:update_environment:Function]\n # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.\n def update_environment(self, env_id: str, env: Environment) -> bool:\n with belief_scope(\"ConfigManager.update_environment\", f\"env_id={env_id}\"):\n for index, existing in enumerate(self.config.environments):\n if existing.id != env_id:\n continue\n\n update_data = env.model_dump()\n if update_data.get(\"password\") == \"********\":\n update_data[\"password\"] = existing.password\n\n updated = Environment.model_validate(update_data)\n\n if updated.is_default:\n for item in self.config.environments:\n item.is_default = False\n elif existing.is_default and not updated.is_default:\n updated.is_default = True\n\n self.config.environments[index] = updated\n log.reason(\"Environment updated\", payload={\"env_id\": env_id})\n self.save()\n return True\n\n log.explore(\"Environment update skipped; env not found\", error=f\"No matching environment found for id: {env_id}\", payload={\"env_id\": env_id})\n return False\n\n # [/DEF:update_environment:Function]\n" }, { "contract_id": "delete_environment", "contract_type": "Function", "file_path": "backend/src/core/config_manager.py", - "start_line": 561, - "end_line": 594, + "start_line": 542, + "end_line": 572, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -32205,7 +32205,7 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:delete_environment:Function]\n # @PURPOSE: Delete environment by id and persist when deletion occurs.\n def delete_environment(self, env_id: str) -> bool:\n with belief_scope(\"ConfigManager.delete_environment\", f\"env_id={env_id}\"):\n before = len(self.config.environments)\n removed = [env for env in self.config.environments if env.id == env_id]\n self.config.environments = [\n env for env in self.config.environments if env.id != env_id\n ]\n\n if len(self.config.environments) == before:\n logger.explore(\n \"Environment delete skipped; env not found\",\n extra={\"env_id\": env_id},\n )\n return False\n\n if removed and removed[0].is_default and self.config.environments:\n self.config.environments[0].is_default = True\n\n if self.config.settings.default_environment_id == env_id:\n replacement = next(\n (env.id for env in self.config.environments if env.is_default), None\n )\n self.config.settings.default_environment_id = replacement\n\n logger.reason(\n \"Environment deleted\",\n extra={\"env_id\": env_id, \"remaining\": len(self.config.environments)},\n )\n self.save()\n return True\n\n # [/DEF:delete_environment:Function]\n" + "body": " # [DEF:delete_environment:Function]\n # @PURPOSE: Delete environment by id and persist when deletion occurs.\n def delete_environment(self, env_id: str) -> bool:\n with belief_scope(\"ConfigManager.delete_environment\", f\"env_id={env_id}\"):\n before = len(self.config.environments)\n removed = [env for env in self.config.environments if env.id == env_id]\n self.config.environments = [\n env for env in self.config.environments if env.id != env_id\n ]\n\n if len(self.config.environments) == before:\n log.explore(\"Environment delete skipped; env not found\", error=f\"No matching environment found for id: {env_id}\", payload={\"env_id\": env_id})\n return False\n\n if removed and removed[0].is_default and self.config.environments:\n self.config.environments[0].is_default = True\n\n if self.config.settings.default_environment_id == env_id:\n replacement = next(\n (env.id for env in self.config.environments if env.is_default), None\n )\n self.config.settings.default_environment_id = replacement\n\n log.reason(\n \"Environment deleted\",\n payload={\"env_id\": env_id, \"remaining\": len(self.config.environments)},\n )\n self.save()\n return True\n\n # [/DEF:delete_environment:Function]\n" }, { "contract_id": "ConfigModels", @@ -32603,6 +32603,800 @@ "anchor_syntax": "def", "body": "# [DEF:AppConfig:DataClass]\n# @PURPOSE: The root configuration model containing all application settings.\nclass AppConfig(BaseModel):\n environments: List[Environment] = []\n settings: GlobalSettings\n\n\n# [/DEF:AppConfig:DataClass]\n" }, + { + "contract_id": "CotLoggerModule", + "contract_type": "Module", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 1, + "end_line": 258, + "tier": "TIER_2", + "complexity": 4, + "metadata": { + "COMPLEXITY": 4, + "PURPOSE": "Structured JSON logger implementing the molecular CoT (Chain-of-Thought) logging protocol.", + "SEMANTICS": [ + "logging", + "cot", + "molecular", + "structured", + "json", + "contextvar" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "missing_required_tag", + "tag": "LAYER", + "message": "@LAYER is required for contract type 'Module' at C4", + "detail": { + "actual_complexity": 4, + "contract_type": "Module" + } + }, + { + "code": "missing_required_tag", + "tag": "POST", + "message": "@POST is required for contract type 'Module' at C4", + "detail": { + "actual_complexity": 4, + "contract_type": "Module" + } + }, + { + "code": "missing_required_tag", + "tag": "PRE", + "message": "@PRE is required for contract type 'Module' at C4", + "detail": { + "actual_complexity": 4, + "contract_type": "Module" + } + }, + { + "code": "missing_required_tag", + "tag": "RELATION", + "message": "@RELATION is required for contract type 'Module' at C4", + "detail": { + "actual_complexity": 4, + "contract_type": "Module" + } + }, + { + "code": "missing_required_tag", + "tag": "SIDE_EFFECT", + "message": "@SIDE_EFFECT is required for contract type 'Module' at C4", + "detail": { + "actual_complexity": 4, + "contract_type": "Module" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:CotLoggerModule:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: logging, cot, molecular, structured, json, contextvar\n# @PURPOSE: Structured JSON logger implementing the molecular CoT (Chain-of-Thought) logging protocol.\n# Uses ContextVar for trace_id and span_id propagation across async contexts.\n# Provides log(), MarkerLogger, seed_trace_id(), push_span(), pop_span().\n# @LAYER: Core\n# @RELATION: [CALLED_BY] -> [TraceContextMiddleware]\n# @RELATION: [CALLED_BY] -> [All C4+ service and route modules]\n# @PRE: Python 3.7+ (ContextVar available).\n# @POST: JSON log records written to the 'cot' logger at appropriate levels.\n# @SIDE_EFFECT: Writes structured JSON to the 'cot' Python logger.\n# @DATA_CONTRACT: Log call -> Single-line JSON to logging.StreamHandler/file.\n\nimport json\nimport logging\nimport uuid\nfrom contextvars import ContextVar\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, Optional\n\n# [DEF:cot_trace_context:Data]\n# @COMPLEXITY: 1\n# @SEMANTICS: contextvar, trace_id, span_id, propagation\n# @PURPOSE: ContextVars for trace ID and span ID propagation across async boundaries.\n_trace_id: ContextVar[str] = ContextVar(\"_trace_id\", default=\"\")\n_span_id: ContextVar[str] = ContextVar(\"_span_id\", default=\"\")\n# [/DEF:cot_trace_context:Data]\n\n# [DEF:cot_logger_instance:Data]\n# @COMPLEXITY: 1\n# @SEMANTICS: logger, instance\n# @PURPOSE: Dedicated Python logger for all CoT (molecular) log output.\ncot_logger = logging.getLogger(\"cot\")\n# [/DEF:cot_logger_instance:Data]\n\n__all__ = [\n \"seed_trace_id\",\n \"set_trace_id\",\n \"get_trace_id\",\n \"push_span\",\n \"pop_span\",\n \"log\",\n \"MarkerLogger\",\n]\n\n# [DEF:seed_trace_id:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: trace_id, uuid, contextvar, set\n# @BRIEF: Generate a new UUID4 trace_id, set it in ContextVar, and return it.\ndef seed_trace_id() -> str:\n \"\"\"Generate a new UUID4 trace ID and store it in the thread-local ContextVar.\n\n Returns:\n str: The newly generated trace ID.\n \"\"\"\n trace_id = str(uuid.uuid4())\n _trace_id.set(trace_id)\n return trace_id\n\n\n# [/DEF:seed_trace_id:Function]\n\n# [DEF:set_trace_id:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: trace_id, contextvar, set, public\n# @BRIEF: Set an explicit trace_id into the ContextVar (e.g. from an incoming header).\ndef set_trace_id(trace_id: str) -> None:\n \"\"\"Set a specific trace_id into the ContextVar.\n\n This is used by TraceContextMiddleware when an ``X-Trace-ID`` header\n is present, enabling cross-service trace chaining.\n\n Args:\n trace_id: The trace ID to set.\n \"\"\"\n _trace_id.set(trace_id)\n\n\n# [/DEF:set_trace_id:Function]\n\n# [DEF:get_trace_id:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: trace_id, contextvar, get, public\n# @BRIEF: Get the current trace_id from ContextVar.\ndef get_trace_id() -> str:\n \"\"\"Get the current trace_id from the thread-local ContextVar.\n\n Returns:\n str: The current trace ID, or empty string if none set.\n \"\"\"\n return _trace_id.get()\n\n\n# [/DEF:get_trace_id:Function]\n\n# [DEF:push_span:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: span_id, contextvar, stack\n# @BRIEF: Set a new span_id in ContextVar and return the previous span_id for later restoration.\ndef push_span(span: str) -> str:\n \"\"\"Push a new span ID onto the context and return the previous span ID.\n\n Args:\n span: The new span identifier (e.g. function or operation name).\n\n Returns:\n str: The previous span ID, suitable for passing to pop_span().\n \"\"\"\n prev = _span_id.get()\n _span_id.set(span)\n return prev\n\n\n# [/DEF:push_span:Function]\n\n# [DEF:pop_span:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: span_id, contextvar, restore\n# @BRIEF: Restore a previous span_id into the ContextVar.\ndef pop_span(prev: str) -> None:\n \"\"\"Restore a previous span ID into the ContextVar.\n\n Args:\n prev: The span ID to restore (previously returned by push_span).\n \"\"\"\n _span_id.set(prev)\n\n\n# [/DEF:pop_span:Function]\n\n# [DEF:cot_log_function:Function]\n# @COMPLEXITY: 2\n# @SEMANTICS: log, json, structured, marker\n# @BRIEF: Core structured logging function that emits a single-line JSON record.\ndef log(\n src: str,\n marker: str,\n intent: str,\n payload: Optional[Dict[str, Any]] = None,\n error: Optional[str] = None,\n level: Optional[str] = None,\n) -> None:\n \"\"\"Emit a single-line structured JSON log record through the 'cot' logger.\n\n Args:\n src: Qualified function or component name (e.g. 'AuthRepository.get_user').\n marker: Protocol marker — one of 'REASON', 'REFLECT', 'EXPLORE'.\n intent: Short one-line description of the intent or action.\n payload: Optional structured data dict to include in the record.\n error: Required for EXPLORE markers; describes the violated assumption.\n level: Log level override. Auto-inferred from marker if omitted\n (REASON/REFLECT -> INFO, EXPLORE -> WARNING).\n\n Side effects:\n Writes a single-line JSON string to the 'cot' Python logger.\n \"\"\"\n if level is None:\n level = \"WARNING\" if marker == \"EXPLORE\" else \"INFO\"\n\n # Build structured extra data for the LogRecord.\n # CotJsonFormatter will read these attributes to produce the final JSON output,\n # consistent with the main superset_tools_app logger.\n extra: Dict[str, Any] = {\n \"marker\": marker,\n \"intent\": intent,\n \"src\": src,\n }\n\n if payload is not None:\n extra[\"payload\"] = payload\n\n if error is not None:\n extra[\"error\"] = error\n\n log_func = {\n \"WARNING\": cot_logger.warning,\n \"ERROR\": cot_logger.error,\n \"DEBUG\": cot_logger.debug,\n }.get(level, cot_logger.info)\n\n log_func(intent, extra=extra)\n\n\n# [/DEF:cot_log_function:Function]\n\n# [DEF:MarkerLogger:Class]\n# @COMPLEXITY: 2\n# @SEMANTICS: logger, proxy, marker, syntactic-sugar\n# @BRIEF: Thin proxy class providing .reason(), .reflect(), .explore() convenience methods.\nclass MarkerLogger:\n \"\"\"Thin proxy over the cot_logger.log() function.\n\n Usage::\n\n log = MarkerLogger(\"AuthService\")\n log.reason(\"Validating credentials\", payload={\"user_id\": \"...\"})\n log.reflect(\"Validation complete\", payload={\"result\": \"ok\"})\n log.explore(\"Token expired\", error=\"JWT expired, attempting refresh\")\n\n Each method delegates to :func:`log` with the corresponding marker.\n \"\"\"\n\n # [DEF:MarkerLogger.__init__:Function]\n # @BRIEF: Store the module/component name used as the 'src' field in all log calls.\n def __init__(self, module_name: str) -> None:\n \"\"\"Initialise the MarkerLogger with a module or component name.\n\n Args:\n module_name: The value used for the 'src' field (e.g. 'AuthService').\n \"\"\"\n self._module_name = module_name\n\n # [/DEF:MarkerLogger.__init__:Function]\n\n # [DEF:MarkerLogger.reason:Function]\n # @BRIEF: Log a REASON marker — strict deduction, core logic.\n def reason(\n self, intent: str, *, payload: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Log a REASON marker (INFO level).\"\"\"\n log(self._module_name, \"REASON\", intent, payload=payload)\n\n # [/DEF:MarkerLogger.reason:Function]\n\n # [DEF:MarkerLogger.reflect:Function]\n # @BRIEF: Log a REFLECT marker — self-check, structural validation.\n def reflect(\n self, intent: str, *, payload: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Log a REFLECT marker (INFO level).\"\"\"\n log(self._module_name, \"REFLECT\", intent, payload=payload)\n\n # [/DEF:MarkerLogger.reflect:Function]\n\n # [DEF:MarkerLogger.explore:Function]\n # @BRIEF: Log an EXPLORE marker — searching, alternatives, violated assumptions.\n def explore(\n self,\n intent: str,\n *,\n payload: Optional[Dict[str, Any]] = None,\n error: Optional[str] = None,\n ) -> None:\n \"\"\"Log an EXPLORE marker (WARNING level).\n\n Args:\n intent: Short description of what was being attempted.\n payload: Optional structured data.\n error: Describes the violated assumption or error context.\n \"\"\"\n log(self._module_name, \"EXPLORE\", intent, payload=payload, error=error)\n\n # [/DEF:MarkerLogger.explore:Function]\n\n\n# [/DEF:MarkerLogger:Class]\n# [/DEF:CotLoggerModule:Module]\n" + }, + { + "contract_id": "cot_trace_context", + "contract_type": "Data", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 22, + "end_line": 28, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1, + "PURPOSE": "ContextVars for trace ID and span ID propagation across async boundaries.", + "SEMANTICS": [ + "contextvar", + "trace_id", + "span_id", + "propagation" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "tag_not_for_contract_type", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is not allowed for contract type 'Data'", + "detail": { + "actual_type": "Data", + "allowed_types": [ + "Module", + "Function", + "Class", + "Component", + "Block" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is forbidden for contract type 'Data' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Data" + } + }, + { + "code": "tag_not_for_contract_type", + "tag": "PURPOSE", + "message": "@PURPOSE is not allowed for contract type 'Data'", + "detail": { + "actual_type": "Data", + "allowed_types": [ + "Module", + "Function", + "Class", + "Component", + "Block", + "ADR" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "PURPOSE", + "message": "@PURPOSE is forbidden for contract type 'Data' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Data" + } + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Data'", + "detail": { + "actual_type": "Data", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Data' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Data" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:cot_trace_context:Data]\n# @COMPLEXITY: 1\n# @SEMANTICS: contextvar, trace_id, span_id, propagation\n# @PURPOSE: ContextVars for trace ID and span ID propagation across async boundaries.\n_trace_id: ContextVar[str] = ContextVar(\"_trace_id\", default=\"\")\n_span_id: ContextVar[str] = ContextVar(\"_span_id\", default=\"\")\n# [/DEF:cot_trace_context:Data]\n" + }, + { + "contract_id": "cot_logger_instance", + "contract_type": "Data", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 30, + "end_line": 35, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1, + "PURPOSE": "Dedicated Python logger for all CoT (molecular) log output.", + "SEMANTICS": [ + "logger", + "instance" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "tag_not_for_contract_type", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is not allowed for contract type 'Data'", + "detail": { + "actual_type": "Data", + "allowed_types": [ + "Module", + "Function", + "Class", + "Component", + "Block" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is forbidden for contract type 'Data' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Data" + } + }, + { + "code": "tag_not_for_contract_type", + "tag": "PURPOSE", + "message": "@PURPOSE is not allowed for contract type 'Data'", + "detail": { + "actual_type": "Data", + "allowed_types": [ + "Module", + "Function", + "Class", + "Component", + "Block", + "ADR" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "PURPOSE", + "message": "@PURPOSE is forbidden for contract type 'Data' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Data" + } + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Data'", + "detail": { + "actual_type": "Data", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Data' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Data" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:cot_logger_instance:Data]\n# @COMPLEXITY: 1\n# @SEMANTICS: logger, instance\n# @PURPOSE: Dedicated Python logger for all CoT (molecular) log output.\ncot_logger = logging.getLogger(\"cot\")\n# [/DEF:cot_logger_instance:Data]\n" + }, + { + "contract_id": "seed_trace_id", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 47, + "end_line": 62, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Generate a new UUID4 trace_id, set it in ContextVar, and return it.", + "COMPLEXITY": 1, + "SEMANTICS": [ + "trace_id", + "uuid", + "contextvar", + "set" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:seed_trace_id:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: trace_id, uuid, contextvar, set\n# @BRIEF: Generate a new UUID4 trace_id, set it in ContextVar, and return it.\ndef seed_trace_id() -> str:\n \"\"\"Generate a new UUID4 trace ID and store it in the thread-local ContextVar.\n\n Returns:\n str: The newly generated trace ID.\n \"\"\"\n trace_id = str(uuid.uuid4())\n _trace_id.set(trace_id)\n return trace_id\n\n\n# [/DEF:seed_trace_id:Function]\n" + }, + { + "contract_id": "set_trace_id", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 64, + "end_line": 80, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Set an explicit trace_id into the ContextVar (e.g. from an incoming header).", + "COMPLEXITY": 1, + "SEMANTICS": [ + "trace_id", + "contextvar", + "set", + "public" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:set_trace_id:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: trace_id, contextvar, set, public\n# @BRIEF: Set an explicit trace_id into the ContextVar (e.g. from an incoming header).\ndef set_trace_id(trace_id: str) -> None:\n \"\"\"Set a specific trace_id into the ContextVar.\n\n This is used by TraceContextMiddleware when an ``X-Trace-ID`` header\n is present, enabling cross-service trace chaining.\n\n Args:\n trace_id: The trace ID to set.\n \"\"\"\n _trace_id.set(trace_id)\n\n\n# [/DEF:set_trace_id:Function]\n" + }, + { + "contract_id": "get_trace_id", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 82, + "end_line": 95, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Get the current trace_id from ContextVar.", + "COMPLEXITY": 1, + "SEMANTICS": [ + "trace_id", + "contextvar", + "get", + "public" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:get_trace_id:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: trace_id, contextvar, get, public\n# @BRIEF: Get the current trace_id from ContextVar.\ndef get_trace_id() -> str:\n \"\"\"Get the current trace_id from the thread-local ContextVar.\n\n Returns:\n str: The current trace ID, or empty string if none set.\n \"\"\"\n return _trace_id.get()\n\n\n# [/DEF:get_trace_id:Function]\n" + }, + { + "contract_id": "push_span", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 97, + "end_line": 115, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Set a new span_id in ContextVar and return the previous span_id for later restoration.", + "COMPLEXITY": 1, + "SEMANTICS": [ + "span_id", + "contextvar", + "stack" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:push_span:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: span_id, contextvar, stack\n# @BRIEF: Set a new span_id in ContextVar and return the previous span_id for later restoration.\ndef push_span(span: str) -> str:\n \"\"\"Push a new span ID onto the context and return the previous span ID.\n\n Args:\n span: The new span identifier (e.g. function or operation name).\n\n Returns:\n str: The previous span ID, suitable for passing to pop_span().\n \"\"\"\n prev = _span_id.get()\n _span_id.set(span)\n return prev\n\n\n# [/DEF:push_span:Function]\n" + }, + { + "contract_id": "pop_span", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 117, + "end_line": 130, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Restore a previous span_id into the ContextVar.", + "COMPLEXITY": 1, + "SEMANTICS": [ + "span_id", + "contextvar", + "restore" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:pop_span:Function]\n# @COMPLEXITY: 1\n# @SEMANTICS: span_id, contextvar, restore\n# @BRIEF: Restore a previous span_id into the ContextVar.\ndef pop_span(prev: str) -> None:\n \"\"\"Restore a previous span ID into the ContextVar.\n\n Args:\n prev: The span ID to restore (previously returned by push_span).\n \"\"\"\n _span_id.set(prev)\n\n\n# [/DEF:pop_span:Function]\n" + }, + { + "contract_id": "cot_log_function", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 132, + "end_line": 185, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Core structured logging function that emits a single-line JSON record.", + "COMPLEXITY": 2, + "SEMANTICS": [ + "log", + "json", + "structured", + "marker" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:cot_log_function:Function]\n# @COMPLEXITY: 2\n# @SEMANTICS: log, json, structured, marker\n# @BRIEF: Core structured logging function that emits a single-line JSON record.\ndef log(\n src: str,\n marker: str,\n intent: str,\n payload: Optional[Dict[str, Any]] = None,\n error: Optional[str] = None,\n level: Optional[str] = None,\n) -> None:\n \"\"\"Emit a single-line structured JSON log record through the 'cot' logger.\n\n Args:\n src: Qualified function or component name (e.g. 'AuthRepository.get_user').\n marker: Protocol marker — one of 'REASON', 'REFLECT', 'EXPLORE'.\n intent: Short one-line description of the intent or action.\n payload: Optional structured data dict to include in the record.\n error: Required for EXPLORE markers; describes the violated assumption.\n level: Log level override. Auto-inferred from marker if omitted\n (REASON/REFLECT -> INFO, EXPLORE -> WARNING).\n\n Side effects:\n Writes a single-line JSON string to the 'cot' Python logger.\n \"\"\"\n if level is None:\n level = \"WARNING\" if marker == \"EXPLORE\" else \"INFO\"\n\n # Build structured extra data for the LogRecord.\n # CotJsonFormatter will read these attributes to produce the final JSON output,\n # consistent with the main superset_tools_app logger.\n extra: Dict[str, Any] = {\n \"marker\": marker,\n \"intent\": intent,\n \"src\": src,\n }\n\n if payload is not None:\n extra[\"payload\"] = payload\n\n if error is not None:\n extra[\"error\"] = error\n\n log_func = {\n \"WARNING\": cot_logger.warning,\n \"ERROR\": cot_logger.error,\n \"DEBUG\": cot_logger.debug,\n }.get(level, cot_logger.info)\n\n log_func(intent, extra=extra)\n\n\n# [/DEF:cot_log_function:Function]\n" + }, + { + "contract_id": "MarkerLogger", + "contract_type": "Class", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 187, + "end_line": 257, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Thin proxy class providing .reason(), .reflect(), .explore() convenience methods.", + "COMPLEXITY": 2, + "SEMANTICS": [ + "logger", + "proxy", + "marker", + "syntactic-sugar" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Class'", + "detail": { + "actual_type": "Class", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Class' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Class" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Class' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Class" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:MarkerLogger:Class]\n# @COMPLEXITY: 2\n# @SEMANTICS: logger, proxy, marker, syntactic-sugar\n# @BRIEF: Thin proxy class providing .reason(), .reflect(), .explore() convenience methods.\nclass MarkerLogger:\n \"\"\"Thin proxy over the cot_logger.log() function.\n\n Usage::\n\n log = MarkerLogger(\"AuthService\")\n log.reason(\"Validating credentials\", payload={\"user_id\": \"...\"})\n log.reflect(\"Validation complete\", payload={\"result\": \"ok\"})\n log.explore(\"Token expired\", error=\"JWT expired, attempting refresh\")\n\n Each method delegates to :func:`log` with the corresponding marker.\n \"\"\"\n\n # [DEF:MarkerLogger.__init__:Function]\n # @BRIEF: Store the module/component name used as the 'src' field in all log calls.\n def __init__(self, module_name: str) -> None:\n \"\"\"Initialise the MarkerLogger with a module or component name.\n\n Args:\n module_name: The value used for the 'src' field (e.g. 'AuthService').\n \"\"\"\n self._module_name = module_name\n\n # [/DEF:MarkerLogger.__init__:Function]\n\n # [DEF:MarkerLogger.reason:Function]\n # @BRIEF: Log a REASON marker — strict deduction, core logic.\n def reason(\n self, intent: str, *, payload: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Log a REASON marker (INFO level).\"\"\"\n log(self._module_name, \"REASON\", intent, payload=payload)\n\n # [/DEF:MarkerLogger.reason:Function]\n\n # [DEF:MarkerLogger.reflect:Function]\n # @BRIEF: Log a REFLECT marker — self-check, structural validation.\n def reflect(\n self, intent: str, *, payload: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Log a REFLECT marker (INFO level).\"\"\"\n log(self._module_name, \"REFLECT\", intent, payload=payload)\n\n # [/DEF:MarkerLogger.reflect:Function]\n\n # [DEF:MarkerLogger.explore:Function]\n # @BRIEF: Log an EXPLORE marker — searching, alternatives, violated assumptions.\n def explore(\n self,\n intent: str,\n *,\n payload: Optional[Dict[str, Any]] = None,\n error: Optional[str] = None,\n ) -> None:\n \"\"\"Log an EXPLORE marker (WARNING level).\n\n Args:\n intent: Short description of what was being attempted.\n payload: Optional structured data.\n error: Describes the violated assumption or error context.\n \"\"\"\n log(self._module_name, \"EXPLORE\", intent, payload=payload, error=error)\n\n # [/DEF:MarkerLogger.explore:Function]\n\n\n# [/DEF:MarkerLogger:Class]\n" + }, + { + "contract_id": "MarkerLogger.__init__", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 204, + "end_line": 214, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Store the module/component name used as the 'src' field in all log calls." + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": " # [DEF:MarkerLogger.__init__:Function]\n # @BRIEF: Store the module/component name used as the 'src' field in all log calls.\n def __init__(self, module_name: str) -> None:\n \"\"\"Initialise the MarkerLogger with a module or component name.\n\n Args:\n module_name: The value used for the 'src' field (e.g. 'AuthService').\n \"\"\"\n self._module_name = module_name\n\n # [/DEF:MarkerLogger.__init__:Function]\n" + }, + { + "contract_id": "MarkerLogger.reason", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 216, + "end_line": 224, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Log a REASON marker — strict deduction, core logic." + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": " # [DEF:MarkerLogger.reason:Function]\n # @BRIEF: Log a REASON marker — strict deduction, core logic.\n def reason(\n self, intent: str, *, payload: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Log a REASON marker (INFO level).\"\"\"\n log(self._module_name, \"REASON\", intent, payload=payload)\n\n # [/DEF:MarkerLogger.reason:Function]\n" + }, + { + "contract_id": "MarkerLogger.reflect", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 226, + "end_line": 234, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Log a REFLECT marker — self-check, structural validation." + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": " # [DEF:MarkerLogger.reflect:Function]\n # @BRIEF: Log a REFLECT marker — self-check, structural validation.\n def reflect(\n self, intent: str, *, payload: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Log a REFLECT marker (INFO level).\"\"\"\n log(self._module_name, \"REFLECT\", intent, payload=payload)\n\n # [/DEF:MarkerLogger.reflect:Function]\n" + }, + { + "contract_id": "MarkerLogger.explore", + "contract_type": "Function", + "file_path": "backend/src/core/cot_logger.py", + "start_line": 236, + "end_line": 254, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Log an EXPLORE marker — searching, alternatives, violated assumptions." + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": " # [DEF:MarkerLogger.explore:Function]\n # @BRIEF: Log an EXPLORE marker — searching, alternatives, violated assumptions.\n def explore(\n self,\n intent: str,\n *,\n payload: Optional[Dict[str, Any]] = None,\n error: Optional[str] = None,\n ) -> None:\n \"\"\"Log an EXPLORE marker (WARNING level).\n\n Args:\n intent: Short description of what was being attempted.\n payload: Optional structured data.\n error: Describes the violated assumption or error context.\n \"\"\"\n log(self._module_name, \"EXPLORE\", intent, payload=payload, error=error)\n\n # [/DEF:MarkerLogger.explore:Function]\n" + }, { "contract_id": "DatabaseModule", "contract_type": "Module", @@ -34233,28 +35027,48 @@ "contract_type": "Module", "file_path": "backend/src/core/logger.py", "start_line": 1, - "end_line": 319, - "tier": "TIER_1", - "complexity": 1, + "end_line": 343, + "tier": "TIER_3", + "complexity": 5, "metadata": { + "BRIEF": "Application logging system with CotJsonFormatter producing molecular CoT JSON output.", + "COMPLEXITY": 5, + "DATA_CONTRACT": "All log records conform to molecular CoT JSON schema: {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?}", + "INVARIANT": "CotJsonFormatter.format() always returns valid single-line JSON string.", "LAYER": "Core", - "PURPOSE": "Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets.", - "SEMANTICS": [ - "logging", - "websocket", - "streaming", - "handler" - ] + "POST": "All log output is single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.", + "PRE": "Python 3.7+ with cot_logger ContextVars available.", + "RATIONALE": "Replaced BeliefFormatter text-based [Entry]/[Exit]/[Action] logging with CotJsonFormatter JSON output. Old format was not machine-parseable and mixed presentation with intent. CotJsonFormatter produces structured JSON consistent with cot_logger.py MarkerLogger protocol.", + "REJECTED": "Keeping both formatters side-by-side was rejected (two inconsistent output formats would confuse log consumers).", + "SIDE_EFFECT": "Configures global logger handlers and formatters; seeds global _enable_belief_state and _task_log_level; writes JSON to console/files." }, "relations": [ + { + "source_id": "LoggerModule", + "relation_type": "USED_BY", + "target_id": "All application modules", + "target_ref": "[All application modules]" + }, { "source_id": "LoggerModule", "relation_type": "DEPENDS_ON", - "target_id": "Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py.", - "target_ref": "Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py." + "target_id": "CotLoggerModule", + "target_ref": "[CotLoggerModule]" + }, + { + "source_id": "LoggerModule", + "relation_type": "DEPENDS_ON", + "target_id": "WebSocketLogHandler", + "target_ref": "[WebSocketLogHandler]" } ], "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, { "code": "invalid_enum_value", "tag": "LAYER", @@ -34270,570 +35084,240 @@ } }, { - "code": "tag_forbidden_by_complexity", - "tag": "RELATION", - "message": "@RELATION is forbidden for contract type 'Module' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Module' at C5", "detail": { - "actual_complexity": 1, + "actual_complexity": 5, "contract_type": "Module" } - } - ], - "anchor_syntax": "def", - "body": "# [DEF:LoggerModule:Module]\n# @SEMANTICS: logging, websocket, streaming, handler\n# @PURPOSE: Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets.\n# @LAYER: Core\n# @RELATION: Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py.\nimport logging\nimport threading\nfrom datetime import datetime\nfrom typing import Dict, Any, List, Optional\nfrom collections import deque\nfrom contextlib import contextmanager\nfrom logging.handlers import RotatingFileHandler\n\nfrom pydantic import BaseModel, Field\n\n# Thread-local storage for belief state\n_belief_state = threading.local()\n\n# Global flag for belief state logging\n_enable_belief_state = True\n\n# Global task log level filter\n_task_log_level = \"INFO\"\n\n# [DEF:BeliefFormatter:Class]\n# @PURPOSE: Custom logging formatter that adds belief state prefixes to log messages.\nclass BeliefFormatter(logging.Formatter):\n # [DEF:format:Function]\n # @PURPOSE: Formats the log record, adding belief state context if available.\n # @PRE: record is a logging.LogRecord.\n # @POST: Returns formatted string.\n # @PARAM: record (logging.LogRecord) - The log record to format.\n # @RETURN: str - The formatted log message.\n # @SEMANTICS: logging, formatter, context\n def format(self, record):\n anchor_id = getattr(_belief_state, 'anchor_id', None)\n if anchor_id:\n msg = str(record.msg)\n # Supported molecular topology markers\n markers = (\"[EXPLORE]\", \"[REASON]\", \"[REFLECT]\", \"[COHERENCE:\", \"[Action]\", \"[Entry]\", \"[Exit]\")\n \n # Avoid duplicating anchor or overriding explicit markers\n if msg.startswith(f\"[{anchor_id}]\"):\n pass\n elif any(msg.startswith(m) for m in markers):\n record.msg = f\"[{anchor_id}]{msg}\"\n else:\n # Default covalent bond\n record.msg = f\"[{anchor_id}][Action] {msg}\"\n \n return super().format(record)\n # [/DEF:format:Function]\n# [/DEF:BeliefFormatter:Class]\n\n# Re-using LogEntry from task_manager for consistency\n# [DEF:LogEntry:Class]\n# @SEMANTICS: log, entry, record, pydantic\n# @PURPOSE: A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py.\nclass LogEntry(BaseModel):\n timestamp: datetime = Field(default_factory=datetime.utcnow)\n level: str\n message: str\n context: Optional[Dict[str, Any]] = None\n\n# [/DEF:LogEntry:Class]\n\n# [DEF:belief_scope:Function]\n# @PURPOSE: Context manager for structured Belief State logging.\n# @PARAM: anchor_id (str) - The identifier for the current semantic block.\n# @PARAM: message (str) - Optional entry message.\n# @PRE: anchor_id must be provided.\n# @POST: Thread-local belief state is updated and entry/exit logs are generated.\n# @SEMANTICS: logging, context, belief_state\n@contextmanager\ndef belief_scope(anchor_id: str, message: str = \"\"):\n # Log Entry if enabled (DEBUG level to reduce noise)\n if _enable_belief_state:\n entry_msg = f\"[{anchor_id}][Entry]\"\n if message:\n entry_msg += f\" {message}\"\n logger.debug(entry_msg)\n\n # Set thread-local anchor_id\n old_anchor = getattr(_belief_state, 'anchor_id', None)\n _belief_state.anchor_id = anchor_id\n\n try:\n yield\n # Log Coherence OK and Exit (DEBUG level to reduce noise)\n logger.debug(\"[COHERENCE:OK]\")\n if _enable_belief_state:\n logger.debug(\"[Exit]\")\n except Exception as e:\n # Log Coherence Failed (DEBUG level to reduce noise)\n logger.debug(f\"[COHERENCE:FAILED] {str(e)}\")\n raise\n finally:\n # Restore old anchor\n _belief_state.anchor_id = old_anchor\n\n# [/DEF:belief_scope:Function]\n\n# [DEF:configure_logger:Function]\n# @PURPOSE: Configures the logger with the provided logging settings.\n# @PRE: config is a valid LoggingConfig instance.\n# @POST: Logger level, handlers, belief state flag, and task log level are updated.\n# @PARAM: config (LoggingConfig) - The logging configuration.\n# @SEMANTICS: logging, configuration, initialization\ndef configure_logger(config):\n global _enable_belief_state, _task_log_level\n _enable_belief_state = config.enable_belief_state\n _task_log_level = config.task_log_level.upper()\n\n # Set logger level\n level = getattr(logging, config.level.upper(), logging.INFO)\n logger.setLevel(level)\n\n # Remove existing file handlers\n handlers_to_remove = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n for h in handlers_to_remove:\n logger.removeHandler(h)\n h.close()\n\n # Add file handler if file_path is set\n if config.file_path:\n from pathlib import Path\n log_file = Path(config.file_path)\n log_file.parent.mkdir(parents=True, exist_ok=True)\n \n file_handler = RotatingFileHandler(\n config.file_path,\n maxBytes=config.max_bytes,\n backupCount=config.backup_count\n )\n file_handler.setFormatter(BeliefFormatter(\n '[%(asctime)s][%(levelname)s][%(name)s] %(message)s'\n ))\n logger.addHandler(file_handler)\n\n # Update existing handlers' formatters to BeliefFormatter\n for handler in logger.handlers:\n if not isinstance(handler, RotatingFileHandler):\n handler.setFormatter(BeliefFormatter(\n '[%(asctime)s][%(levelname)s][%(name)s] %(message)s'\n ))\n# [/DEF:configure_logger:Function]\n\n# [DEF:get_task_log_level:Function]\n# @PURPOSE: Returns the current task log level filter.\n# @PRE: None.\n# @POST: Returns the task log level string.\n# @RETURN: str - The current task log level (DEBUG, INFO, WARNING, ERROR).\n# @SEMANTICS: logging, configuration, getter\ndef get_task_log_level() -> str:\n \"\"\"Returns the current task log level filter.\"\"\"\n return _task_log_level\n# [/DEF:get_task_log_level:Function]\n\n# [DEF:should_log_task_level:Function]\n# @PURPOSE: Checks if a log level should be recorded based on task_log_level setting.\n# @PRE: level is a valid log level string.\n# @POST: Returns True if level meets or exceeds task_log_level threshold.\n# @PARAM: level (str) - The log level to check.\n# @RETURN: bool - True if the level should be logged.\n# @SEMANTICS: logging, filter, level\ndef should_log_task_level(level: str) -> bool:\n \"\"\"Checks if a log level should be recorded based on task_log_level setting.\"\"\"\n level_order = {\"DEBUG\": 0, \"INFO\": 1, \"WARNING\": 2, \"ERROR\": 3}\n current_level = _task_log_level.upper()\n check_level = level.upper()\n \n current_order = level_order.get(current_level, 1) # Default to INFO\n check_order = level_order.get(check_level, 1)\n \n return check_order >= current_order\n# [/DEF:should_log_task_level:Function]\n\n# [DEF:WebSocketLogHandler:Class]\n# @SEMANTICS: logging, handler, websocket, buffer\n# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.\nclass WebSocketLogHandler(logging.Handler):\n \"\"\"\n A logging handler that stores log records and can be extended to send them\n over WebSockets.\n \"\"\"\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the handler with a fixed-capacity buffer.\n # @PRE: capacity is an integer.\n # @POST: Instance initialized with empty deque.\n # @PARAM: capacity (int) - Maximum number of logs to keep in memory.\n # @SEMANTICS: logging, initialization, buffer\n def __init__(self, capacity: int = 1000):\n super().__init__()\n self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)\n # In a real implementation, you'd have a way to manage active WebSocket connections\n # e.g., self.active_connections: Set[WebSocket] = set()\n # [/DEF:__init__:Function]\n\n # [DEF:emit:Function]\n # @PURPOSE: Captures a log record, formats it, and stores it in the buffer.\n # @PRE: record is a logging.LogRecord.\n # @POST: Log is added to the log_buffer.\n # @PARAM: record (logging.LogRecord) - The log record to emit.\n # @SEMANTICS: logging, handler, buffer\n def emit(self, record: logging.LogRecord):\n try:\n log_entry = LogEntry(\n level=record.levelname,\n message=self.format(record),\n context={\n \"name\": record.name,\n \"pathname\": record.pathname,\n \"lineno\": record.lineno,\n \"funcName\": record.funcName,\n \"process\": record.process,\n \"thread\": record.thread,\n }\n )\n self.log_buffer.append(log_entry)\n # Here you would typically send the log_entry to all active WebSocket connections\n # for real-time streaming to the frontend.\n # Example: for ws in self.active_connections: await ws.send_json(log_entry.dict())\n except Exception:\n self.handleError(record)\n # [/DEF:emit:Function]\n\n # [DEF:get_recent_logs:Function]\n # @PURPOSE: Returns a list of recent log entries from the buffer.\n # @PRE: None.\n # @POST: Returns list of LogEntry objects.\n # @RETURN: List[LogEntry] - List of buffered log entries.\n # @SEMANTICS: logging, buffer, retrieval\n def get_recent_logs(self) -> List[LogEntry]:\n \"\"\"\n Returns a list of recent log entries from the buffer.\n \"\"\"\n return list(self.log_buffer)\n # [/DEF:get_recent_logs:Function]\n\n# [/DEF:WebSocketLogHandler:Class]\n\n# [DEF:Logger:Global]\n# @SEMANTICS: logger, global, instance\n# @PURPOSE: The global logger instance for the application, configured with both a console handler and the custom WebSocket handler.\nlogger = logging.getLogger(\"superset_tools_app\")\n\n# [DEF:believed:Function]\n# @PURPOSE: A decorator that wraps a function in a belief scope.\n# @PARAM: anchor_id (str) - The identifier for the semantic block.\n# @PRE: anchor_id must be a string.\n# @POST: Returns a decorator function.\ndef believed(anchor_id: str):\n # [DEF:decorator:Function]\n # @PURPOSE: Internal decorator for belief scope.\n # @PRE: func must be a callable.\n # @POST: Returns the wrapped function.\n def decorator(func):\n # [DEF:wrapper:Function]\n # @PURPOSE: Internal wrapper that enters belief scope.\n # @PRE: None.\n # @POST: Executes the function within a belief scope.\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # [/DEF:wrapper:Function]\n return wrapper\n # [/DEF:decorator:Function]\n return decorator\n# [/DEF:believed:Function]\nlogger.setLevel(logging.INFO)\n\n# Create a formatter\nformatter = BeliefFormatter(\n '[%(asctime)s][%(levelname)s][%(name)s] %(message)s'\n)\n\n# Add console handler\nconsole_handler = logging.StreamHandler()\nconsole_handler.setFormatter(formatter)\nlogger.addHandler(console_handler)\n\n# Add WebSocket log handler\nwebsocket_log_handler = WebSocketLogHandler()\nwebsocket_log_handler.setFormatter(formatter)\nlogger.addHandler(websocket_log_handler)\n\n# Example usage:\n# logger.info(\"Application started\", extra={\"context_key\": \"context_value\"})\n# logger.error(\"An error occurred\", exc_info=True)\n\nimport types\n\n# [DEF:explore:Function]\n# @PURPOSE: Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses.\n# @SEMANTICS: log, explore, molecule\ndef explore(self, msg, *args, **kwargs):\n self.warning(f\"[EXPLORE] {msg}\", *args, **kwargs)\n# [/DEF:explore:Function]\n\n# [DEF:reason:Function]\n# @PURPOSE: Logs a REASON message (Covalent bond) for strict deduction and core logic.\n# @SEMANTICS: log, reason, molecule\ndef reason(self, msg, *args, **kwargs):\n self.info(f\"[REASON] {msg}\", *args, **kwargs)\n# [/DEF:reason:Function]\n\n# [DEF:reflect:Function]\n# @PURPOSE: Logs a REFLECT message (Hydrogen bond) for self-check and structural validation.\n# @SEMANTICS: log, reflect, molecule\ndef reflect(self, msg, *args, **kwargs):\n self.debug(f\"[REFLECT] {msg}\", *args, **kwargs)\n# [/DEF:reflect:Function]\n\nlogger.explore = types.MethodType(explore, logger)\nlogger.reason = types.MethodType(reason, logger)\nlogger.reflect = types.MethodType(reflect, logger)\n\n# [/DEF:Logger:Global]\n# [/DEF:LoggerModule:Module]\n" - }, - { - "contract_id": "BeliefFormatter", - "contract_type": "Class", - "file_path": "backend/src/core/logger.py", - "start_line": 25, - "end_line": 53, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "PURPOSE": "Custom logging formatter that adds belief state prefixes to log messages." - }, - "relations": [], - "schema_warnings": [], - "anchor_syntax": "def", - "body": "# [DEF:BeliefFormatter:Class]\n# @PURPOSE: Custom logging formatter that adds belief state prefixes to log messages.\nclass BeliefFormatter(logging.Formatter):\n # [DEF:format:Function]\n # @PURPOSE: Formats the log record, adding belief state context if available.\n # @PRE: record is a logging.LogRecord.\n # @POST: Returns formatted string.\n # @PARAM: record (logging.LogRecord) - The log record to format.\n # @RETURN: str - The formatted log message.\n # @SEMANTICS: logging, formatter, context\n def format(self, record):\n anchor_id = getattr(_belief_state, 'anchor_id', None)\n if anchor_id:\n msg = str(record.msg)\n # Supported molecular topology markers\n markers = (\"[EXPLORE]\", \"[REASON]\", \"[REFLECT]\", \"[COHERENCE:\", \"[Action]\", \"[Entry]\", \"[Exit]\")\n \n # Avoid duplicating anchor or overriding explicit markers\n if msg.startswith(f\"[{anchor_id}]\"):\n pass\n elif any(msg.startswith(m) for m in markers):\n record.msg = f\"[{anchor_id}]{msg}\"\n else:\n # Default covalent bond\n record.msg = f\"[{anchor_id}][Action] {msg}\"\n \n return super().format(record)\n # [/DEF:format:Function]\n# [/DEF:BeliefFormatter:Class]\n" - }, - { - "contract_id": "format", - "contract_type": "Function", - "file_path": "backend/src/core/logger.py", - "start_line": 28, - "end_line": 52, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "PARAM": "record (logging.LogRecord) - The log record to format.", - "POST": "Returns formatted string.", - "PRE": "record is a logging.LogRecord.", - "PURPOSE": "Formats the log record, adding belief state context if available.", - "RETURN": "str - The formatted log message.", - "SEMANTICS": [ - "logging", - "formatter", - "context" - ] - }, - "relations": [], - "schema_warnings": [ - { - "code": "unknown_tag", - "tag": "PARAM", - "message": "@PARAM is not defined in axiom_config.yaml tags", - "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", + "code": "invalid_relation_predicate", + "tag": "RELATION", + "message": "Predicate USED_BY is not in allowed_predicates", "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "unknown_tag", - "tag": "RETURN", - "message": "@RETURN is not defined in axiom_config.yaml tags", - "detail": null - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" + "allowed": [ + "DEPENDS_ON", + "CALLS", + "INHERITS", + "IMPLEMENTS", + "DISPATCHES", + "BINDS_TO", + "VERIFIES" + ], + "predicate": "USED_BY" } } ], - "anchor_syntax": "def", - "body": " # [DEF:format:Function]\n # @PURPOSE: Formats the log record, adding belief state context if available.\n # @PRE: record is a logging.LogRecord.\n # @POST: Returns formatted string.\n # @PARAM: record (logging.LogRecord) - The log record to format.\n # @RETURN: str - The formatted log message.\n # @SEMANTICS: logging, formatter, context\n def format(self, record):\n anchor_id = getattr(_belief_state, 'anchor_id', None)\n if anchor_id:\n msg = str(record.msg)\n # Supported molecular topology markers\n markers = (\"[EXPLORE]\", \"[REASON]\", \"[REFLECT]\", \"[COHERENCE:\", \"[Action]\", \"[Entry]\", \"[Exit]\")\n \n # Avoid duplicating anchor or overriding explicit markers\n if msg.startswith(f\"[{anchor_id}]\"):\n pass\n elif any(msg.startswith(m) for m in markers):\n record.msg = f\"[{anchor_id}]{msg}\"\n else:\n # Default covalent bond\n record.msg = f\"[{anchor_id}][Action] {msg}\"\n \n return super().format(record)\n # [/DEF:format:Function]\n" + "anchor_syntax": "region", + "body": "# #region LoggerModule [C:5] [TYPE Module] [SEMANTICS logging,websocket,streaming,handler,cot,json]\n# @BRIEF Application logging system with CotJsonFormatter producing molecular CoT JSON output.\n# @LAYER Core\n# @RELATION USED_BY -> [All application modules]\n# @RELATION DEPENDS_ON -> [CotLoggerModule]\n# @RELATION DEPENDS_ON -> [WebSocketLogHandler]\n# @PRE Python 3.7+ with cot_logger ContextVars available.\n# @POST All log output is single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.\n# @SIDE_EFFECT Configures global logger handlers and formatters; seeds global _enable_belief_state and _task_log_level; writes JSON to console/files.\n# @RATIONALE Replaced BeliefFormatter text-based [Entry]/[Exit]/[Action] logging with CotJsonFormatter JSON output. Old format was not machine-parseable and mixed presentation with intent. CotJsonFormatter produces structured JSON consistent with cot_logger.py MarkerLogger protocol.\n# @REJECTED Keeping both formatters side-by-side was rejected (two inconsistent output formats would confuse log consumers).\n# @DATA_CONTRACT All log records conform to molecular CoT JSON schema: {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?}\n# @INVARIANT CotJsonFormatter.format() always returns valid single-line JSON string.\nimport json\nimport logging\nimport threading\nimport types\nfrom datetime import datetime, timezone\nfrom contextlib import contextmanager\nfrom logging.handlers import RotatingFileHandler\n\nfrom .cot_logger import _trace_id, _span_id\nfrom .ws_log_handler import WebSocketLogHandler, LogEntry # noqa: F401\n\n# Thread-local storage for belief state\n_belief_state = threading.local()\n\n# Global flag for belief state logging\n_enable_belief_state = True\n\n# Global task log level filter\n_task_log_level = \"INFO\"\n\n\n# #region CotJsonFormatter [C:3] [TYPE Class] [SEMANTICS logging,formatter,json,cot,protocol]\n# @BRIEF JSON formatter implementing the molecular CoT logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.\n# @RELATION IMPLEMENTS -> [CotJsonFormat]\nclass CotJsonFormatter(logging.Formatter):\n \"\"\"JSON formatter implementing the molecular CoT logging protocol.\n\n Reads structured data from the LogRecord's extra attributes (marker, intent, payload,\n error, src) set via the ``extra`` kwarg. Falls back to plain message wrapping when no\n structured extra is present.\n\n Output format (single-line JSON)::\n\n {\n \"ts\": \"2026-05-12T10:30:00.123\",\n \"level\": \"INFO\",\n \"trace_id\": \"uuid-or-no-trace\",\n \"src\": \"module.name\",\n \"marker\": \"REASON|REFLECT|EXPLORE\",\n \"intent\": \"human-readable intent\",\n \"span_id\": \"optional-span\",\n \"payload\": { ... }, # optional\n \"error\": \"...\" # optional\n }\n \"\"\"\n\n # #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record]\n # @BRIEF Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields.\n def format(self, record):\n # Try to extract structured data from extra kwargs on the record\n marker = getattr(record, 'marker', None)\n intent = getattr(record, 'intent', None)\n payload = getattr(record, 'payload', None)\n error = getattr(record, 'error', None)\n src = getattr(record, 'src', None) or record.name\n\n # For records that already come as JSON strings (e.g. from cot_logger.log()),\n # detect and pass through the message directly as intent.\n if not marker:\n marker = \"REASON\" # default for plain messages\n if not intent:\n intent = record.getMessage()\n\n log_obj = {\n \"ts\": datetime.now(timezone.utc).isoformat(timespec=\"milliseconds\"),\n \"level\": record.levelname,\n \"trace_id\": _trace_id.get() or \"no-trace\",\n \"src\": src,\n \"marker\": marker,\n \"intent\": intent,\n }\n\n span_id = _span_id.get()\n if span_id:\n log_obj[\"span_id\"] = span_id\n if payload:\n log_obj[\"payload\"] = payload\n if error:\n log_obj[\"error\"] = error\n\n return json.dumps(log_obj, ensure_ascii=False, default=str)\n # #endregion CotJsonFormatter.format\n\n\n# #endregion CotJsonFormatter\n\n\n# #region belief_scope [C:4] [TYPE Function] [SEMANTICS logging,context,belief_state,cot,marker]\n# @BRIEF Context manager for structured Belief State logging with CoT markers (REASON on entry, REFLECT on success, EXPLORE on failure).\n# @RELATION CALLED_BY -> [All C3+ modules using belief_scope]\n# @RELATION BINDS_TO -> [CotJsonFormatter]\n# @PRE anchor_id must be provided.\n# @POST Thread-local belief state is updated. Entry logged as REASON marker, success coherence as REFLECT marker, failure as EXPLORE marker.\n# @SIDE_EFFECT Writes debug-level log records with structured extra data; mutates _belief_state.anchor_id.\n@contextmanager\ndef belief_scope(anchor_id: str, message: str = \"\"):\n \"\"\"Context manager for structured Belief State logging with CoT markers.\n\n Emits structured markers via the ``extra`` dict so that CotJsonFormatter\n produces proper JSON output.\n\n Usage::\n\n with belief_scope(\"MyFunction\", \"Processing request\"):\n logger.info(\"Doing work\")\n \"\"\"\n old_anchor = getattr(_belief_state, 'anchor_id', None)\n _belief_state.anchor_id = anchor_id\n\n # Log Entry (REASON marker) — only when belief state logging is enabled\n if _enable_belief_state:\n entry_msg = message or f\"Enter {anchor_id}\"\n logger.debug(entry_msg, extra={\n \"marker\": \"REASON\",\n \"intent\": entry_msg,\n \"src\": anchor_id,\n })\n\n try:\n yield\n # Coherence OK / scope exit — always logged for internal tracking\n logger.debug(\"Coherence OK\", extra={\n \"marker\": \"REFLECT\",\n \"intent\": \"Coherence OK\",\n \"src\": anchor_id,\n \"payload\": {\"contract_id\": anchor_id},\n })\n except Exception as e:\n # Coherence FAILED — always logged for error tracking\n logger.debug(f\"Coherence FAILED: {e}\", extra={\n \"marker\": \"EXPLORE\",\n \"intent\": \"Coherence FAILED\",\n \"error\": str(e),\n \"src\": anchor_id,\n \"payload\": {\"contract_id\": anchor_id},\n })\n raise\n finally:\n # Restore old anchor\n _belief_state.anchor_id = old_anchor\n\n# #endregion belief_scope\n\n\n# #region configure_logger [C:4] [TYPE Function] [SEMANTICS logging,configuration,initialization,cot]\n# @BRIEF Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags.\n# @RELATION CALLS -> [CotJsonFormatter]\n# @RELATION CALLS -> [CotLoggerModule]\n# @PRE config is a valid LoggingConfig instance.\n# @POST Logger levels, handlers, formatters, belief state flag, and task log level are updated. The 'cot' logger has CotJsonFormatter with propagation disabled.\n# @SIDE_EFFECT Mutates global _enable_belief_state, _task_log_level; adds/removes handlers on both logger and cot_logger; creates file directories.\ndef configure_logger(config):\n global _enable_belief_state, _task_log_level\n _enable_belief_state = config.enable_belief_state\n _task_log_level = config.task_log_level.upper()\n\n # Set logger level\n level = getattr(logging, config.level.upper(), logging.INFO)\n logger.setLevel(level)\n\n # Remove existing file handlers\n handlers_to_remove = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n for h in handlers_to_remove:\n logger.removeHandler(h)\n h.close()\n\n # Add file handler if file_path is set\n if config.file_path:\n from pathlib import Path\n log_file = Path(config.file_path)\n log_file.parent.mkdir(parents=True, exist_ok=True)\n\n file_handler = RotatingFileHandler(\n config.file_path,\n maxBytes=config.max_bytes,\n backupCount=config.backup_count\n )\n file_handler.setFormatter(CotJsonFormatter())\n logger.addHandler(file_handler)\n\n # Update existing handlers' formatters to CotJsonFormatter\n for handler in logger.handlers:\n if not isinstance(handler, RotatingFileHandler):\n handler.setFormatter(CotJsonFormatter())\n\n # Also configure the 'cot' logger for consistent JSON output\n from .cot_logger import cot_logger\n cot_logger.setLevel(level)\n # Remove all existing handlers from the cot logger\n for h in list(cot_logger.handlers):\n cot_logger.removeHandler(h)\n h.close()\n # Add a console handler with CotJsonFormatter (if file path set, add file handler too)\n cot_console = logging.StreamHandler()\n cot_console.setFormatter(CotJsonFormatter())\n cot_logger.addHandler(cot_console)\n if config.file_path:\n from pathlib import Path\n log_file = Path(config.file_path)\n log_file.parent.mkdir(parents=True, exist_ok=True)\n cot_file = RotatingFileHandler(\n config.file_path,\n maxBytes=config.max_bytes,\n backupCount=config.backup_count\n )\n cot_file.setFormatter(CotJsonFormatter())\n cot_logger.addHandler(cot_file)\n # Disable propagation so cot messages don't double-emit via root loggers\n cot_logger.propagate = False\n\n# #endregion configure_logger\n\n\n# #region get_task_log_level [C:1] [TYPE Function] [SEMANTICS logging,configuration,getter]\ndef get_task_log_level() -> str:\n \"\"\"Returns the current task log level filter.\"\"\"\n return _task_log_level\n\n# #endregion get_task_log_level\n\n\n# #region should_log_task_level [C:2] [TYPE Function] [SEMANTICS logging,filter,level]\n# @BRIEF Checks if a log level should be recorded based on the current task log level threshold.\ndef should_log_task_level(level: str) -> bool:\n \"\"\"Checks if a log level should be recorded based on task_log_level setting.\"\"\"\n level_order = {\"DEBUG\": 0, \"INFO\": 1, \"WARNING\": 2, \"ERROR\": 3}\n current_level = _task_log_level.upper()\n check_level = level.upper()\n\n current_order = level_order.get(current_level, 1) # Default to INFO\n check_order = level_order.get(check_level, 1)\n\n return check_order >= current_order\n\n# #endregion should_log_task_level\n\n\n# #region Logger [C:2] [TYPE Global] [SEMANTICS logger,global,instance]\n# @BRIEF The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler.\nlogger = logging.getLogger(\"superset_tools_app\")\nlogger.setLevel(logging.INFO)\n\n# Create CotJsonFormatter\n_formatter = CotJsonFormatter()\n\n# Add console handler\nconsole_handler = logging.StreamHandler()\nconsole_handler.setFormatter(_formatter)\nlogger.addHandler(console_handler)\n\n# Add WebSocket log handler\nwebsocket_log_handler = WebSocketLogHandler()\nwebsocket_log_handler.setFormatter(_formatter)\nlogger.addHandler(websocket_log_handler)\n\n# Example usage:\n# logger.info(\"Application started\", extra={\"context_key\": \"context_value\"})\n# logger.error(\"An error occurred\", exc_info=True)\n\n\n# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning]\n# @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter.\ndef explore(self, msg, *args, **kwargs):\n \"\"\"Log an EXPLORE marker — searching, alternatives, violated assumptions.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'EXPLORE', 'intent': msg}\n extra.update(user_extra)\n self.warning(msg, *args, extra=extra, **kwargs)\n\n# #endregion explore\n\n\n# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info]\n# @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter.\ndef reason(self, msg, *args, **kwargs):\n \"\"\"Log a REASON marker — strict deduction, core logic.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'REASON', 'intent': msg}\n extra.update(user_extra)\n self.info(msg, *args, extra=extra, **kwargs)\n\n# #endregion reason\n\n\n# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,debug]\n# @BRIEF Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter.\ndef reflect(self, msg, *args, **kwargs):\n \"\"\"Log a REFLECT marker — self-check, structural validation.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'REFLECT', 'intent': msg}\n extra.update(user_extra)\n self.debug(msg, *args, extra=extra, **kwargs)\n\n# #endregion reflect\n\n\nlogger.explore = types.MethodType(explore, logger)\nlogger.reason = types.MethodType(reason, logger)\nlogger.reflect = types.MethodType(reflect, logger)\n\n# #endregion Logger\n\n\n# #region believed [C:2] [TYPE Function] [SEMANTICS decorator,belief_scope,wrapper]\n# @BRIEF Decorator that wraps a function in a belief_scope context manager.\ndef believed(anchor_id: str):\n # #region believed.decorator [C:1] [TYPE Function] [SEMANTICS decorator,inner]\n def decorator(func):\n # #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope]\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # #endregion believed.decorator.wrapper\n return wrapper\n # #endregion believed.decorator\n return decorator\n\n# #endregion believed\n\n\n# #endregion LoggerModule\n" }, { - "contract_id": "LogEntry", + "contract_id": "CotJsonFormatter", "contract_type": "Class", "file_path": "backend/src/core/logger.py", - "start_line": 56, - "end_line": 65, - "tier": "TIER_1", - "complexity": 1, + "start_line": 35, + "end_line": 98, + "tier": "TIER_2", + "complexity": 3, "metadata": { - "PURPOSE": "A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py.", - "SEMANTICS": [ - "log", - "entry", - "record", - "pydantic" - ] + "BRIEF": "JSON formatter implementing the molecular CoT logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.", + "COMPLEXITY": 3 }, - "relations": [], + "relations": [ + { + "source_id": "CotJsonFormatter", + "relation_type": "IMPLEMENTS", + "target_id": "CotJsonFormat", + "target_ref": "[CotJsonFormat]" + } + ], "schema_warnings": [ { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Class'", - "detail": { - "actual_type": "Class", - "allowed_types": [ - "Module" - ] - } + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Class' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Class' at C3", "detail": { - "actual_complexity": 1, + "actual_complexity": 3, "contract_type": "Class" } } ], - "anchor_syntax": "def", - "body": "# [DEF:LogEntry:Class]\n# @SEMANTICS: log, entry, record, pydantic\n# @PURPOSE: A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py.\nclass LogEntry(BaseModel):\n timestamp: datetime = Field(default_factory=datetime.utcnow)\n level: str\n message: str\n context: Optional[Dict[str, Any]] = None\n\n# [/DEF:LogEntry:Class]\n" + "anchor_syntax": "region", + "body": "# #region CotJsonFormatter [C:3] [TYPE Class] [SEMANTICS logging,formatter,json,cot,protocol]\n# @BRIEF JSON formatter implementing the molecular CoT logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.\n# @RELATION IMPLEMENTS -> [CotJsonFormat]\nclass CotJsonFormatter(logging.Formatter):\n \"\"\"JSON formatter implementing the molecular CoT logging protocol.\n\n Reads structured data from the LogRecord's extra attributes (marker, intent, payload,\n error, src) set via the ``extra`` kwarg. Falls back to plain message wrapping when no\n structured extra is present.\n\n Output format (single-line JSON)::\n\n {\n \"ts\": \"2026-05-12T10:30:00.123\",\n \"level\": \"INFO\",\n \"trace_id\": \"uuid-or-no-trace\",\n \"src\": \"module.name\",\n \"marker\": \"REASON|REFLECT|EXPLORE\",\n \"intent\": \"human-readable intent\",\n \"span_id\": \"optional-span\",\n \"payload\": { ... }, # optional\n \"error\": \"...\" # optional\n }\n \"\"\"\n\n # #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record]\n # @BRIEF Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields.\n def format(self, record):\n # Try to extract structured data from extra kwargs on the record\n marker = getattr(record, 'marker', None)\n intent = getattr(record, 'intent', None)\n payload = getattr(record, 'payload', None)\n error = getattr(record, 'error', None)\n src = getattr(record, 'src', None) or record.name\n\n # For records that already come as JSON strings (e.g. from cot_logger.log()),\n # detect and pass through the message directly as intent.\n if not marker:\n marker = \"REASON\" # default for plain messages\n if not intent:\n intent = record.getMessage()\n\n log_obj = {\n \"ts\": datetime.now(timezone.utc).isoformat(timespec=\"milliseconds\"),\n \"level\": record.levelname,\n \"trace_id\": _trace_id.get() or \"no-trace\",\n \"src\": src,\n \"marker\": marker,\n \"intent\": intent,\n }\n\n span_id = _span_id.get()\n if span_id:\n log_obj[\"span_id\"] = span_id\n if payload:\n log_obj[\"payload\"] = payload\n if error:\n log_obj[\"error\"] = error\n\n return json.dumps(log_obj, ensure_ascii=False, default=str)\n # #endregion CotJsonFormatter.format\n\n\n# #endregion CotJsonFormatter\n" + }, + { + "contract_id": "CotJsonFormatter.format", + "contract_type": "Function", + "file_path": "backend/src/core/logger.py", + "start_line": 60, + "end_line": 95, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields.", + "COMPLEXITY": 2 + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": " # #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record]\n # @BRIEF Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields.\n def format(self, record):\n # Try to extract structured data from extra kwargs on the record\n marker = getattr(record, 'marker', None)\n intent = getattr(record, 'intent', None)\n payload = getattr(record, 'payload', None)\n error = getattr(record, 'error', None)\n src = getattr(record, 'src', None) or record.name\n\n # For records that already come as JSON strings (e.g. from cot_logger.log()),\n # detect and pass through the message directly as intent.\n if not marker:\n marker = \"REASON\" # default for plain messages\n if not intent:\n intent = record.getMessage()\n\n log_obj = {\n \"ts\": datetime.now(timezone.utc).isoformat(timespec=\"milliseconds\"),\n \"level\": record.levelname,\n \"trace_id\": _trace_id.get() or \"no-trace\",\n \"src\": src,\n \"marker\": marker,\n \"intent\": intent,\n }\n\n span_id = _span_id.get()\n if span_id:\n log_obj[\"span_id\"] = span_id\n if payload:\n log_obj[\"payload\"] = payload\n if error:\n log_obj[\"error\"] = error\n\n return json.dumps(log_obj, ensure_ascii=False, default=str)\n # #endregion CotJsonFormatter.format\n" }, { "contract_id": "configure_logger", "contract_type": "Function", "file_path": "backend/src/core/logger.py", - "start_line": 103, - "end_line": 146, - "tier": "TIER_1", - "complexity": 1, + "start_line": 158, + "end_line": 224, + "tier": "TIER_2", + "complexity": 4, "metadata": { - "PARAM": "config (LoggingConfig) - The logging configuration.", - "POST": "Logger level, handlers, belief state flag, and task log level are updated.", + "BRIEF": "Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags.", + "COMPLEXITY": 4, + "POST": "Logger levels, handlers, formatters, belief state flag, and task log level are updated. The 'cot' logger has CotJsonFormatter with propagation disabled.", "PRE": "config is a valid LoggingConfig instance.", - "PURPOSE": "Configures the logger with the provided logging settings.", - "SEMANTICS": [ - "logging", - "configuration", - "initialization" - ] + "SIDE_EFFECT": "Mutates global _enable_belief_state, _task_log_level; adds/removes handlers on both logger and cot_logger; creates file directories." }, - "relations": [], + "relations": [ + { + "source_id": "configure_logger", + "relation_type": "CALLS", + "target_id": "CotJsonFormatter", + "target_ref": "[CotJsonFormatter]" + }, + { + "source_id": "configure_logger", + "relation_type": "CALLS", + "target_id": "CotLoggerModule", + "target_ref": "[CotLoggerModule]" + } + ], "schema_warnings": [ { "code": "unknown_tag", - "tag": "PARAM", - "message": "@PARAM is not defined in axiom_config.yaml tags", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C4", "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, + "actual_complexity": 4, "contract_type": "Function" } } ], - "anchor_syntax": "def", - "body": "# [DEF:configure_logger:Function]\n# @PURPOSE: Configures the logger with the provided logging settings.\n# @PRE: config is a valid LoggingConfig instance.\n# @POST: Logger level, handlers, belief state flag, and task log level are updated.\n# @PARAM: config (LoggingConfig) - The logging configuration.\n# @SEMANTICS: logging, configuration, initialization\ndef configure_logger(config):\n global _enable_belief_state, _task_log_level\n _enable_belief_state = config.enable_belief_state\n _task_log_level = config.task_log_level.upper()\n\n # Set logger level\n level = getattr(logging, config.level.upper(), logging.INFO)\n logger.setLevel(level)\n\n # Remove existing file handlers\n handlers_to_remove = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n for h in handlers_to_remove:\n logger.removeHandler(h)\n h.close()\n\n # Add file handler if file_path is set\n if config.file_path:\n from pathlib import Path\n log_file = Path(config.file_path)\n log_file.parent.mkdir(parents=True, exist_ok=True)\n \n file_handler = RotatingFileHandler(\n config.file_path,\n maxBytes=config.max_bytes,\n backupCount=config.backup_count\n )\n file_handler.setFormatter(BeliefFormatter(\n '[%(asctime)s][%(levelname)s][%(name)s] %(message)s'\n ))\n logger.addHandler(file_handler)\n\n # Update existing handlers' formatters to BeliefFormatter\n for handler in logger.handlers:\n if not isinstance(handler, RotatingFileHandler):\n handler.setFormatter(BeliefFormatter(\n '[%(asctime)s][%(levelname)s][%(name)s] %(message)s'\n ))\n# [/DEF:configure_logger:Function]\n" + "anchor_syntax": "region", + "body": "# #region configure_logger [C:4] [TYPE Function] [SEMANTICS logging,configuration,initialization,cot]\n# @BRIEF Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags.\n# @RELATION CALLS -> [CotJsonFormatter]\n# @RELATION CALLS -> [CotLoggerModule]\n# @PRE config is a valid LoggingConfig instance.\n# @POST Logger levels, handlers, formatters, belief state flag, and task log level are updated. The 'cot' logger has CotJsonFormatter with propagation disabled.\n# @SIDE_EFFECT Mutates global _enable_belief_state, _task_log_level; adds/removes handlers on both logger and cot_logger; creates file directories.\ndef configure_logger(config):\n global _enable_belief_state, _task_log_level\n _enable_belief_state = config.enable_belief_state\n _task_log_level = config.task_log_level.upper()\n\n # Set logger level\n level = getattr(logging, config.level.upper(), logging.INFO)\n logger.setLevel(level)\n\n # Remove existing file handlers\n handlers_to_remove = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n for h in handlers_to_remove:\n logger.removeHandler(h)\n h.close()\n\n # Add file handler if file_path is set\n if config.file_path:\n from pathlib import Path\n log_file = Path(config.file_path)\n log_file.parent.mkdir(parents=True, exist_ok=True)\n\n file_handler = RotatingFileHandler(\n config.file_path,\n maxBytes=config.max_bytes,\n backupCount=config.backup_count\n )\n file_handler.setFormatter(CotJsonFormatter())\n logger.addHandler(file_handler)\n\n # Update existing handlers' formatters to CotJsonFormatter\n for handler in logger.handlers:\n if not isinstance(handler, RotatingFileHandler):\n handler.setFormatter(CotJsonFormatter())\n\n # Also configure the 'cot' logger for consistent JSON output\n from .cot_logger import cot_logger\n cot_logger.setLevel(level)\n # Remove all existing handlers from the cot logger\n for h in list(cot_logger.handlers):\n cot_logger.removeHandler(h)\n h.close()\n # Add a console handler with CotJsonFormatter (if file path set, add file handler too)\n cot_console = logging.StreamHandler()\n cot_console.setFormatter(CotJsonFormatter())\n cot_logger.addHandler(cot_console)\n if config.file_path:\n from pathlib import Path\n log_file = Path(config.file_path)\n log_file.parent.mkdir(parents=True, exist_ok=True)\n cot_file = RotatingFileHandler(\n config.file_path,\n maxBytes=config.max_bytes,\n backupCount=config.backup_count\n )\n cot_file.setFormatter(CotJsonFormatter())\n cot_logger.addHandler(cot_file)\n # Disable propagation so cot messages don't double-emit via root loggers\n cot_logger.propagate = False\n\n# #endregion configure_logger\n" }, { "contract_id": "get_task_log_level", "contract_type": "Function", "file_path": "backend/src/core/logger.py", - "start_line": 148, - "end_line": 157, + "start_line": 227, + "end_line": 232, "tier": "TIER_1", "complexity": 1, "metadata": { - "POST": "Returns the task log level string.", - "PRE": "None.", - "PURPOSE": "Returns the current task log level filter.", - "RETURN": "str - The current task log level (DEBUG, INFO, WARNING, ERROR).", - "SEMANTICS": [ - "logging", - "configuration", - "getter" - ] + "COMPLEXITY": 1 }, "relations": [], "schema_warnings": [ { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "unknown_tag", - "tag": "RETURN", - "message": "@RETURN is not defined in axiom_config.yaml tags", - "detail": null - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", "detail": { "actual_complexity": 1, "contract_type": "Function" } } ], - "anchor_syntax": "def", - "body": "# [DEF:get_task_log_level:Function]\n# @PURPOSE: Returns the current task log level filter.\n# @PRE: None.\n# @POST: Returns the task log level string.\n# @RETURN: str - The current task log level (DEBUG, INFO, WARNING, ERROR).\n# @SEMANTICS: logging, configuration, getter\ndef get_task_log_level() -> str:\n \"\"\"Returns the current task log level filter.\"\"\"\n return _task_log_level\n# [/DEF:get_task_log_level:Function]\n" + "anchor_syntax": "region", + "body": "# #region get_task_log_level [C:1] [TYPE Function] [SEMANTICS logging,configuration,getter]\ndef get_task_log_level() -> str:\n \"\"\"Returns the current task log level filter.\"\"\"\n return _task_log_level\n\n# #endregion get_task_log_level\n" }, { "contract_id": "should_log_task_level", "contract_type": "Function", "file_path": "backend/src/core/logger.py", - "start_line": 159, - "end_line": 176, + "start_line": 235, + "end_line": 248, "tier": "TIER_1", - "complexity": 1, + "complexity": 2, "metadata": { - "PARAM": "level (str) - The log level to check.", - "POST": "Returns True if level meets or exceeds task_log_level threshold.", - "PRE": "level is a valid log level string.", - "PURPOSE": "Checks if a log level should be recorded based on task_log_level setting.", - "RETURN": "bool - True if the level should be logged.", - "SEMANTICS": [ - "logging", - "filter", - "level" - ] + "BRIEF": "Checks if a log level should be recorded based on the current task log level threshold.", + "COMPLEXITY": 2 }, "relations": [], "schema_warnings": [ { "code": "unknown_tag", - "tag": "PARAM", - "message": "@PARAM is not defined in axiom_config.yaml tags", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "unknown_tag", - "tag": "RETURN", - "message": "@RETURN is not defined in axiom_config.yaml tags", - "detail": null - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, + "actual_complexity": 2, "contract_type": "Function" } } ], - "anchor_syntax": "def", - "body": "# [DEF:should_log_task_level:Function]\n# @PURPOSE: Checks if a log level should be recorded based on task_log_level setting.\n# @PRE: level is a valid log level string.\n# @POST: Returns True if level meets or exceeds task_log_level threshold.\n# @PARAM: level (str) - The log level to check.\n# @RETURN: bool - True if the level should be logged.\n# @SEMANTICS: logging, filter, level\ndef should_log_task_level(level: str) -> bool:\n \"\"\"Checks if a log level should be recorded based on task_log_level setting.\"\"\"\n level_order = {\"DEBUG\": 0, \"INFO\": 1, \"WARNING\": 2, \"ERROR\": 3}\n current_level = _task_log_level.upper()\n check_level = level.upper()\n \n current_order = level_order.get(current_level, 1) # Default to INFO\n check_order = level_order.get(check_level, 1)\n \n return check_order >= current_order\n# [/DEF:should_log_task_level:Function]\n" - }, - { - "contract_id": "WebSocketLogHandler", - "contract_type": "Class", - "file_path": "backend/src/core/logger.py", - "start_line": 178, - "end_line": 240, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "PURPOSE": "A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.", - "SEMANTICS": [ - "logging", - "handler", - "websocket", - "buffer" - ] - }, - "relations": [], - "schema_warnings": [ - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Class'", - "detail": { - "actual_type": "Class", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Class' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Class" - } - } - ], - "anchor_syntax": "def", - "body": "# [DEF:WebSocketLogHandler:Class]\n# @SEMANTICS: logging, handler, websocket, buffer\n# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.\nclass WebSocketLogHandler(logging.Handler):\n \"\"\"\n A logging handler that stores log records and can be extended to send them\n over WebSockets.\n \"\"\"\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the handler with a fixed-capacity buffer.\n # @PRE: capacity is an integer.\n # @POST: Instance initialized with empty deque.\n # @PARAM: capacity (int) - Maximum number of logs to keep in memory.\n # @SEMANTICS: logging, initialization, buffer\n def __init__(self, capacity: int = 1000):\n super().__init__()\n self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)\n # In a real implementation, you'd have a way to manage active WebSocket connections\n # e.g., self.active_connections: Set[WebSocket] = set()\n # [/DEF:__init__:Function]\n\n # [DEF:emit:Function]\n # @PURPOSE: Captures a log record, formats it, and stores it in the buffer.\n # @PRE: record is a logging.LogRecord.\n # @POST: Log is added to the log_buffer.\n # @PARAM: record (logging.LogRecord) - The log record to emit.\n # @SEMANTICS: logging, handler, buffer\n def emit(self, record: logging.LogRecord):\n try:\n log_entry = LogEntry(\n level=record.levelname,\n message=self.format(record),\n context={\n \"name\": record.name,\n \"pathname\": record.pathname,\n \"lineno\": record.lineno,\n \"funcName\": record.funcName,\n \"process\": record.process,\n \"thread\": record.thread,\n }\n )\n self.log_buffer.append(log_entry)\n # Here you would typically send the log_entry to all active WebSocket connections\n # for real-time streaming to the frontend.\n # Example: for ws in self.active_connections: await ws.send_json(log_entry.dict())\n except Exception:\n self.handleError(record)\n # [/DEF:emit:Function]\n\n # [DEF:get_recent_logs:Function]\n # @PURPOSE: Returns a list of recent log entries from the buffer.\n # @PRE: None.\n # @POST: Returns list of LogEntry objects.\n # @RETURN: List[LogEntry] - List of buffered log entries.\n # @SEMANTICS: logging, buffer, retrieval\n def get_recent_logs(self) -> List[LogEntry]:\n \"\"\"\n Returns a list of recent log entries from the buffer.\n \"\"\"\n return list(self.log_buffer)\n # [/DEF:get_recent_logs:Function]\n\n# [/DEF:WebSocketLogHandler:Class]\n" - }, - { - "contract_id": "emit", - "contract_type": "Function", - "file_path": "backend/src/core/logger.py", - "start_line": 199, - "end_line": 225, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "PARAM": "record (logging.LogRecord) - The log record to emit.", - "POST": "Log is added to the log_buffer.", - "PRE": "record is a logging.LogRecord.", - "PURPOSE": "Captures a log record, formats it, and stores it in the buffer.", - "SEMANTICS": [ - "logging", - "handler", - "buffer" - ] - }, - "relations": [], - "schema_warnings": [ - { - "code": "unknown_tag", - "tag": "PARAM", - "message": "@PARAM is not defined in axiom_config.yaml tags", - "detail": null - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - } - ], - "anchor_syntax": "def", - "body": " # [DEF:emit:Function]\n # @PURPOSE: Captures a log record, formats it, and stores it in the buffer.\n # @PRE: record is a logging.LogRecord.\n # @POST: Log is added to the log_buffer.\n # @PARAM: record (logging.LogRecord) - The log record to emit.\n # @SEMANTICS: logging, handler, buffer\n def emit(self, record: logging.LogRecord):\n try:\n log_entry = LogEntry(\n level=record.levelname,\n message=self.format(record),\n context={\n \"name\": record.name,\n \"pathname\": record.pathname,\n \"lineno\": record.lineno,\n \"funcName\": record.funcName,\n \"process\": record.process,\n \"thread\": record.thread,\n }\n )\n self.log_buffer.append(log_entry)\n # Here you would typically send the log_entry to all active WebSocket connections\n # for real-time streaming to the frontend.\n # Example: for ws in self.active_connections: await ws.send_json(log_entry.dict())\n except Exception:\n self.handleError(record)\n # [/DEF:emit:Function]\n" - }, - { - "contract_id": "get_recent_logs", - "contract_type": "Function", - "file_path": "backend/src/core/logger.py", - "start_line": 227, - "end_line": 238, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "POST": "Returns list of LogEntry objects.", - "PRE": "None.", - "PURPOSE": "Returns a list of recent log entries from the buffer.", - "RETURN": "List[LogEntry] - List of buffered log entries.", - "SEMANTICS": [ - "logging", - "buffer", - "retrieval" - ] - }, - "relations": [], - "schema_warnings": [ - { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "unknown_tag", - "tag": "RETURN", - "message": "@RETURN is not defined in axiom_config.yaml tags", - "detail": null - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - } - ], - "anchor_syntax": "def", - "body": " # [DEF:get_recent_logs:Function]\n # @PURPOSE: Returns a list of recent log entries from the buffer.\n # @PRE: None.\n # @POST: Returns list of LogEntry objects.\n # @RETURN: List[LogEntry] - List of buffered log entries.\n # @SEMANTICS: logging, buffer, retrieval\n def get_recent_logs(self) -> List[LogEntry]:\n \"\"\"\n Returns a list of recent log entries from the buffer.\n \"\"\"\n return list(self.log_buffer)\n # [/DEF:get_recent_logs:Function]\n" + "anchor_syntax": "region", + "body": "# #region should_log_task_level [C:2] [TYPE Function] [SEMANTICS logging,filter,level]\n# @BRIEF Checks if a log level should be recorded based on the current task log level threshold.\ndef should_log_task_level(level: str) -> bool:\n \"\"\"Checks if a log level should be recorded based on task_log_level setting.\"\"\"\n level_order = {\"DEBUG\": 0, \"INFO\": 1, \"WARNING\": 2, \"ERROR\": 3}\n current_level = _task_log_level.upper()\n check_level = level.upper()\n\n current_order = level_order.get(current_level, 1) # Default to INFO\n check_order = level_order.get(check_level, 1)\n\n return check_order >= current_order\n\n# #endregion should_log_task_level\n" }, { "contract_id": "Logger", "contract_type": "Global", "file_path": "backend/src/core/logger.py", - "start_line": 242, - "end_line": 318, + "start_line": 251, + "end_line": 323, "tier": "TIER_1", - "complexity": 1, + "complexity": 2, "metadata": { - "PURPOSE": "The global logger instance for the application, configured with both a console handler and the custom WebSocket handler.", - "SEMANTICS": [ - "logger", - "global", - "instance" - ] + "BRIEF": "The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler.", + "COMPLEXITY": 2 }, "relations": [], "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, { "code": "tag_not_for_contract_type", - "tag": "PURPOSE", - "message": "@PURPOSE is not allowed for contract type 'Global'", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is not allowed for contract type 'Global'", "detail": { "actual_type": "Global", "allowed_types": [ @@ -34841,300 +35325,219 @@ "Function", "Class", "Component", - "Block", - "ADR" + "Block" ] } }, { "code": "tag_forbidden_by_complexity", - "tag": "PURPOSE", - "message": "@PURPOSE is forbidden for contract type 'Global' at C1", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is forbidden for contract type 'Global' at C2", "detail": { - "actual_complexity": 1, - "contract_type": "Global" - } - }, - { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Global'", - "detail": { - "actual_type": "Global", - "allowed_types": [ - "Module" - ] - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Global' at C1", - "detail": { - "actual_complexity": 1, + "actual_complexity": 2, "contract_type": "Global" } } ], - "anchor_syntax": "def", - "body": "# [DEF:Logger:Global]\n# @SEMANTICS: logger, global, instance\n# @PURPOSE: The global logger instance for the application, configured with both a console handler and the custom WebSocket handler.\nlogger = logging.getLogger(\"superset_tools_app\")\n\n# [DEF:believed:Function]\n# @PURPOSE: A decorator that wraps a function in a belief scope.\n# @PARAM: anchor_id (str) - The identifier for the semantic block.\n# @PRE: anchor_id must be a string.\n# @POST: Returns a decorator function.\ndef believed(anchor_id: str):\n # [DEF:decorator:Function]\n # @PURPOSE: Internal decorator for belief scope.\n # @PRE: func must be a callable.\n # @POST: Returns the wrapped function.\n def decorator(func):\n # [DEF:wrapper:Function]\n # @PURPOSE: Internal wrapper that enters belief scope.\n # @PRE: None.\n # @POST: Executes the function within a belief scope.\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # [/DEF:wrapper:Function]\n return wrapper\n # [/DEF:decorator:Function]\n return decorator\n# [/DEF:believed:Function]\nlogger.setLevel(logging.INFO)\n\n# Create a formatter\nformatter = BeliefFormatter(\n '[%(asctime)s][%(levelname)s][%(name)s] %(message)s'\n)\n\n# Add console handler\nconsole_handler = logging.StreamHandler()\nconsole_handler.setFormatter(formatter)\nlogger.addHandler(console_handler)\n\n# Add WebSocket log handler\nwebsocket_log_handler = WebSocketLogHandler()\nwebsocket_log_handler.setFormatter(formatter)\nlogger.addHandler(websocket_log_handler)\n\n# Example usage:\n# logger.info(\"Application started\", extra={\"context_key\": \"context_value\"})\n# logger.error(\"An error occurred\", exc_info=True)\n\nimport types\n\n# [DEF:explore:Function]\n# @PURPOSE: Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses.\n# @SEMANTICS: log, explore, molecule\ndef explore(self, msg, *args, **kwargs):\n self.warning(f\"[EXPLORE] {msg}\", *args, **kwargs)\n# [/DEF:explore:Function]\n\n# [DEF:reason:Function]\n# @PURPOSE: Logs a REASON message (Covalent bond) for strict deduction and core logic.\n# @SEMANTICS: log, reason, molecule\ndef reason(self, msg, *args, **kwargs):\n self.info(f\"[REASON] {msg}\", *args, **kwargs)\n# [/DEF:reason:Function]\n\n# [DEF:reflect:Function]\n# @PURPOSE: Logs a REFLECT message (Hydrogen bond) for self-check and structural validation.\n# @SEMANTICS: log, reflect, molecule\ndef reflect(self, msg, *args, **kwargs):\n self.debug(f\"[REFLECT] {msg}\", *args, **kwargs)\n# [/DEF:reflect:Function]\n\nlogger.explore = types.MethodType(explore, logger)\nlogger.reason = types.MethodType(reason, logger)\nlogger.reflect = types.MethodType(reflect, logger)\n\n# [/DEF:Logger:Global]\n" - }, - { - "contract_id": "believed", - "contract_type": "Function", - "file_path": "backend/src/core/logger.py", - "start_line": 247, - "end_line": 269, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "PARAM": "anchor_id (str) - The identifier for the semantic block.", - "POST": "Returns a decorator function.", - "PRE": "anchor_id must be a string.", - "PURPOSE": "A decorator that wraps a function in a belief scope." - }, - "relations": [], - "schema_warnings": [ - { - "code": "unknown_tag", - "tag": "PARAM", - "message": "@PARAM is not defined in axiom_config.yaml tags", - "detail": null - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - } - ], - "anchor_syntax": "def", - "body": "# [DEF:believed:Function]\n# @PURPOSE: A decorator that wraps a function in a belief scope.\n# @PARAM: anchor_id (str) - The identifier for the semantic block.\n# @PRE: anchor_id must be a string.\n# @POST: Returns a decorator function.\ndef believed(anchor_id: str):\n # [DEF:decorator:Function]\n # @PURPOSE: Internal decorator for belief scope.\n # @PRE: func must be a callable.\n # @POST: Returns the wrapped function.\n def decorator(func):\n # [DEF:wrapper:Function]\n # @PURPOSE: Internal wrapper that enters belief scope.\n # @PRE: None.\n # @POST: Executes the function within a belief scope.\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # [/DEF:wrapper:Function]\n return wrapper\n # [/DEF:decorator:Function]\n return decorator\n# [/DEF:believed:Function]\n" - }, - { - "contract_id": "decorator", - "contract_type": "Function", - "file_path": "backend/src/core/logger.py", - "start_line": 253, - "end_line": 267, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "POST": "Returns the wrapped function.", - "PRE": "func must be a callable.", - "PURPOSE": "Internal decorator for belief scope." - }, - "relations": [], - "schema_warnings": [ - { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - } - ], - "anchor_syntax": "def", - "body": " # [DEF:decorator:Function]\n # @PURPOSE: Internal decorator for belief scope.\n # @PRE: func must be a callable.\n # @POST: Returns the wrapped function.\n def decorator(func):\n # [DEF:wrapper:Function]\n # @PURPOSE: Internal wrapper that enters belief scope.\n # @PRE: None.\n # @POST: Executes the function within a belief scope.\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # [/DEF:wrapper:Function]\n return wrapper\n # [/DEF:decorator:Function]\n" - }, - { - "contract_id": "wrapper", - "contract_type": "Function", - "file_path": "backend/src/core/logger.py", - "start_line": 258, - "end_line": 265, - "tier": "TIER_1", - "complexity": 1, - "metadata": { - "POST": "Executes the function within a belief scope.", - "PRE": "None.", - "PURPOSE": "Internal wrapper that enters belief scope." - }, - "relations": [], - "schema_warnings": [ - { - "code": "tag_forbidden_by_complexity", - "tag": "POST", - "message": "@POST is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - }, - { - "code": "tag_forbidden_by_complexity", - "tag": "PRE", - "message": "@PRE is forbidden for contract type 'Function' at C1", - "detail": { - "actual_complexity": 1, - "contract_type": "Function" - } - } - ], - "anchor_syntax": "def", - "body": " # [DEF:wrapper:Function]\n # @PURPOSE: Internal wrapper that enters belief scope.\n # @PRE: None.\n # @POST: Executes the function within a belief scope.\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # [/DEF:wrapper:Function]\n" + "anchor_syntax": "region", + "body": "# #region Logger [C:2] [TYPE Global] [SEMANTICS logger,global,instance]\n# @BRIEF The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler.\nlogger = logging.getLogger(\"superset_tools_app\")\nlogger.setLevel(logging.INFO)\n\n# Create CotJsonFormatter\n_formatter = CotJsonFormatter()\n\n# Add console handler\nconsole_handler = logging.StreamHandler()\nconsole_handler.setFormatter(_formatter)\nlogger.addHandler(console_handler)\n\n# Add WebSocket log handler\nwebsocket_log_handler = WebSocketLogHandler()\nwebsocket_log_handler.setFormatter(_formatter)\nlogger.addHandler(websocket_log_handler)\n\n# Example usage:\n# logger.info(\"Application started\", extra={\"context_key\": \"context_value\"})\n# logger.error(\"An error occurred\", exc_info=True)\n\n\n# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning]\n# @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter.\ndef explore(self, msg, *args, **kwargs):\n \"\"\"Log an EXPLORE marker — searching, alternatives, violated assumptions.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'EXPLORE', 'intent': msg}\n extra.update(user_extra)\n self.warning(msg, *args, extra=extra, **kwargs)\n\n# #endregion explore\n\n\n# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info]\n# @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter.\ndef reason(self, msg, *args, **kwargs):\n \"\"\"Log a REASON marker — strict deduction, core logic.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'REASON', 'intent': msg}\n extra.update(user_extra)\n self.info(msg, *args, extra=extra, **kwargs)\n\n# #endregion reason\n\n\n# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,debug]\n# @BRIEF Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter.\ndef reflect(self, msg, *args, **kwargs):\n \"\"\"Log a REFLECT marker — self-check, structural validation.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'REFLECT', 'intent': msg}\n extra.update(user_extra)\n self.debug(msg, *args, extra=extra, **kwargs)\n\n# #endregion reflect\n\n\nlogger.explore = types.MethodType(explore, logger)\nlogger.reason = types.MethodType(reason, logger)\nlogger.reflect = types.MethodType(reflect, logger)\n\n# #endregion Logger\n" }, { "contract_id": "explore", "contract_type": "Function", "file_path": "backend/src/core/logger.py", - "start_line": 293, - "end_line": 298, + "start_line": 274, + "end_line": 286, "tier": "TIER_1", - "complexity": 1, + "complexity": 2, "metadata": { - "PURPOSE": "Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses.", - "SEMANTICS": [ - "log", - "explore", - "molecule" - ] + "BRIEF": "Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter.", + "COMPLEXITY": 2 }, "relations": [], "schema_warnings": [ { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", "detail": { - "actual_complexity": 1, + "actual_complexity": 2, "contract_type": "Function" } } ], - "anchor_syntax": "def", - "body": "# [DEF:explore:Function]\n# @PURPOSE: Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses.\n# @SEMANTICS: log, explore, molecule\ndef explore(self, msg, *args, **kwargs):\n self.warning(f\"[EXPLORE] {msg}\", *args, **kwargs)\n# [/DEF:explore:Function]\n" + "anchor_syntax": "region", + "body": "# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning]\n# @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter.\ndef explore(self, msg, *args, **kwargs):\n \"\"\"Log an EXPLORE marker — searching, alternatives, violated assumptions.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'EXPLORE', 'intent': msg}\n extra.update(user_extra)\n self.warning(msg, *args, extra=extra, **kwargs)\n\n# #endregion explore\n" }, { "contract_id": "reason", "contract_type": "Function", "file_path": "backend/src/core/logger.py", - "start_line": 300, - "end_line": 305, + "start_line": 289, + "end_line": 301, "tier": "TIER_1", - "complexity": 1, + "complexity": 2, "metadata": { - "PURPOSE": "Logs a REASON message (Covalent bond) for strict deduction and core logic.", - "SEMANTICS": [ - "log", - "reason", - "molecule" - ] + "BRIEF": "Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter.", + "COMPLEXITY": 2 }, "relations": [], "schema_warnings": [ { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", "detail": { - "actual_complexity": 1, + "actual_complexity": 2, "contract_type": "Function" } } ], - "anchor_syntax": "def", - "body": "# [DEF:reason:Function]\n# @PURPOSE: Logs a REASON message (Covalent bond) for strict deduction and core logic.\n# @SEMANTICS: log, reason, molecule\ndef reason(self, msg, *args, **kwargs):\n self.info(f\"[REASON] {msg}\", *args, **kwargs)\n# [/DEF:reason:Function]\n" + "anchor_syntax": "region", + "body": "# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info]\n# @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter.\ndef reason(self, msg, *args, **kwargs):\n \"\"\"Log a REASON marker — strict deduction, core logic.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'REASON', 'intent': msg}\n extra.update(user_extra)\n self.info(msg, *args, extra=extra, **kwargs)\n\n# #endregion reason\n" }, { "contract_id": "reflect", "contract_type": "Function", "file_path": "backend/src/core/logger.py", - "start_line": 307, - "end_line": 312, + "start_line": 304, + "end_line": 316, "tier": "TIER_1", - "complexity": 1, + "complexity": 2, "metadata": { - "PURPOSE": "Logs a REFLECT message (Hydrogen bond) for self-check and structural validation.", - "SEMANTICS": [ - "log", - "reflect", - "molecule" - ] + "BRIEF": "Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter.", + "COMPLEXITY": 2 }, "relations": [], "schema_warnings": [ { - "code": "tag_not_for_contract_type", - "tag": "SEMANTICS", - "message": "@SEMANTICS is not allowed for contract type 'Function'", - "detail": { - "actual_type": "Function", - "allowed_types": [ - "Module" - ] - } + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null }, { - "code": "tag_forbidden_by_complexity", - "tag": "SEMANTICS", - "message": "@SEMANTICS is forbidden for contract type 'Function' at C1", + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": "# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,debug]\n# @BRIEF Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter.\ndef reflect(self, msg, *args, **kwargs):\n \"\"\"Log a REFLECT marker — self-check, structural validation.\n\n Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.\n \"\"\"\n user_extra = kwargs.pop('extra', {})\n extra = {'marker': 'REFLECT', 'intent': msg}\n extra.update(user_extra)\n self.debug(msg, *args, extra=extra, **kwargs)\n\n# #endregion reflect\n" + }, + { + "contract_id": "believed", + "contract_type": "Function", + "file_path": "backend/src/core/logger.py", + "start_line": 326, + "end_line": 340, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Decorator that wraps a function in a belief_scope context manager.", + "COMPLEXITY": 2 + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": "# #region believed [C:2] [TYPE Function] [SEMANTICS decorator,belief_scope,wrapper]\n# @BRIEF Decorator that wraps a function in a belief_scope context manager.\ndef believed(anchor_id: str):\n # #region believed.decorator [C:1] [TYPE Function] [SEMANTICS decorator,inner]\n def decorator(func):\n # #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope]\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # #endregion believed.decorator.wrapper\n return wrapper\n # #endregion believed.decorator\n return decorator\n\n# #endregion believed\n" + }, + { + "contract_id": "believed.decorator", + "contract_type": "Function", + "file_path": "backend/src/core/logger.py", + "start_line": 329, + "end_line": 337, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1 + }, + "relations": [], + "schema_warnings": [ + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", "detail": { "actual_complexity": 1, "contract_type": "Function" } } ], - "anchor_syntax": "def", - "body": "# [DEF:reflect:Function]\n# @PURPOSE: Logs a REFLECT message (Hydrogen bond) for self-check and structural validation.\n# @SEMANTICS: log, reflect, molecule\ndef reflect(self, msg, *args, **kwargs):\n self.debug(f\"[REFLECT] {msg}\", *args, **kwargs)\n# [/DEF:reflect:Function]\n" + "anchor_syntax": "region", + "body": " # #region believed.decorator [C:1] [TYPE Function] [SEMANTICS decorator,inner]\n def decorator(func):\n # #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope]\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # #endregion believed.decorator.wrapper\n return wrapper\n # #endregion believed.decorator\n" + }, + { + "contract_id": "believed.decorator.wrapper", + "contract_type": "Function", + "file_path": "backend/src/core/logger.py", + "start_line": 331, + "end_line": 335, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1 + }, + "relations": [], + "schema_warnings": [ + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": " # #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope]\n def wrapper(*args, **kwargs):\n with belief_scope(anchor_id):\n return func(*args, **kwargs)\n # #endregion believed.decorator.wrapper\n" }, { "contract_id": "test_logger", "contract_type": "Module", "file_path": "backend/src/core/logger/__tests__/test_logger.py", "start_line": 1, - "end_line": 291, + "end_line": 400, "tier": "TIER_2", "complexity": 3, "metadata": { "COMPLEXITY": 3, "LAYER": "Infra", - "PURPOSE": "Unit tests for logger module" + "PURPOSE": "Unit tests for logger module — CoT JSON format markers." }, "relations": [ { @@ -35146,24 +35549,24 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:test_logger:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: Unit tests for logger module\n# @LAYER: Infra\n# @RELATION: VERIFIES -> src.core.logger\n\nimport sys\nfrom pathlib import Path\n\n# Add src to path\nsys.path.append(str(Path(__file__).parent.parent.parent.parent / \"src\"))\n\nimport pytest\nimport logging\nfrom src.core.logger import (\n belief_scope,\n logger,\n configure_logger,\n get_task_log_level,\n should_log_task_level\n)\nfrom src.core.config_models import LoggingConfig\n\n\n@pytest.fixture(autouse=True)\ndef reset_logger_state():\n \"\"\"Reset logger state before each test to avoid cross-test contamination.\"\"\"\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n # Also reset the logger level for caplog to work correctly\n logging.getLogger(\"superset_tools_app\").setLevel(logging.DEBUG)\n yield\n # Reset after test too\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n\n# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level.\ndef test_belief_scope_logs_entry_action_exit_at_debug(caplog):\n \"\"\"Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"TestFunction\"):\n logger.info(\"Doing something important\")\n\n # Check that the logs contain the expected patterns\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[TestFunction][Entry]\" in msg for msg in log_messages), \"Entry log not found\"\n assert any(\"[TestFunction][Action] Doing something important\" in msg for msg in log_messages), \"Action log not found\"\n assert any(\"[TestFunction][Exit]\" in msg for msg in log_messages), \"Exit log not found\"\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]\n\n\n# [DEF:test_belief_scope_error_handling:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Coherence:Failed tag.\ndef test_belief_scope_error_handling(caplog):\n \"\"\"Test that belief_scope logs Coherence:Failed on exception.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with pytest.raises(ValueError):\n with belief_scope(\"FailingFunction\"):\n raise ValueError(\"Something went wrong\")\n\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[FailingFunction][Entry]\" in msg for msg in log_messages), \"Entry log not found\"\n assert any(\"[FailingFunction][COHERENCE:FAILED]\" in msg for msg in log_messages), \"Failed coherence log not found\"\n # Exit should not be logged on failure\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_error_handling:Function]\n\n\n# [DEF:test_belief_scope_success_coherence:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs Coherence:OK on success.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Coherence:OK tag.\ndef test_belief_scope_success_coherence(caplog):\n \"\"\"Test that belief_scope logs Coherence:OK on success.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"SuccessFunction\"):\n pass\n\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[SuccessFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Success coherence log not found\"\n \n\n# [/DEF:test_belief_scope_success_coherence:Function]\n\n\n# [DEF:test_belief_scope_not_visible_at_info:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.\n# @PRE: belief_scope is available. caplog fixture is used.\n# @POST: Entry/Exit/Coherence logs are not captured at INFO level.\ndef test_belief_scope_not_visible_at_info(caplog):\n \"\"\"Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.\"\"\"\n caplog.set_level(\"INFO\")\n\n with belief_scope(\"InfoLevelFunction\"):\n logger.info(\"Doing something important\")\n\n log_messages = [record.message for record in caplog.records]\n\n # Action log should be visible\n assert any(\"[InfoLevelFunction][Action] Doing something important\" in msg for msg in log_messages), \"Action log not found\"\n # Entry/Exit/Coherence should NOT be visible at INFO level\n assert not any(\"[InfoLevelFunction][Entry]\" in msg for msg in log_messages), \"Entry log should not be visible at INFO\"\n assert not any(\"[InfoLevelFunction][Exit]\" in msg for msg in log_messages), \"Exit log should not be visible at INFO\"\n assert not any(\"[InfoLevelFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Coherence log should not be visible at INFO\"\n# [/DEF:test_belief_scope_not_visible_at_info:Function]\n\n\n# [DEF:test_task_log_level_default:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that default task log level is INFO.\n# @PRE: None.\n# @POST: Default level is INFO.\ndef test_task_log_level_default():\n \"\"\"Test that default task log level is INFO (after reset fixture).\"\"\"\n level = get_task_log_level()\n assert level == \"INFO\"\n# [/DEF:test_task_log_level_default:Function]\n\n\n# [DEF:test_should_log_task_level:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that should_log_task_level correctly filters log levels.\n# @PRE: None.\n# @POST: Filtering works correctly for all level combinations.\ndef test_should_log_task_level():\n \"\"\"Test that should_log_task_level correctly filters log levels.\"\"\"\n # Default level is INFO\n assert should_log_task_level(\"ERROR\") is True, \"ERROR should be logged at INFO threshold\"\n assert should_log_task_level(\"WARNING\") is True, \"WARNING should be logged at INFO threshold\"\n assert should_log_task_level(\"INFO\") is True, \"INFO should be logged at INFO threshold\"\n assert should_log_task_level(\"DEBUG\") is False, \"DEBUG should NOT be logged at INFO threshold\"\n# [/DEF:test_should_log_task_level:Function]\n\n\n# [DEF:test_configure_logger_task_log_level:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that configure_logger updates task_log_level.\n# @PRE: LoggingConfig is available.\n# @POST: task_log_level is updated correctly.\ndef test_configure_logger_task_log_level():\n \"\"\"Test that configure_logger updates task_log_level.\"\"\"\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n assert get_task_log_level() == \"DEBUG\", \"task_log_level should be DEBUG\"\n assert should_log_task_level(\"DEBUG\") is True, \"DEBUG should be logged at DEBUG threshold\"\n# [/DEF:test_configure_logger_task_log_level:Function]\n\n\n# [DEF:test_enable_belief_state_flag:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging.\n# @PRE: LoggingConfig is available. caplog fixture is used.\n# @POST: belief_scope logs are controlled by the flag.\ndef test_enable_belief_state_flag(caplog):\n \"\"\"Test that enable_belief_state flag controls belief_scope logging.\"\"\"\n # Disable belief state\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n \n with belief_scope(\"DisabledFunction\"):\n logger.info(\"Doing something\")\n \n log_messages = [record.message for record in caplog.records]\n \n # Entry and Exit should NOT be logged when disabled\n assert not any(\"[DisabledFunction][Entry]\" in msg for msg in log_messages), \"Entry should not be logged when disabled\"\n assert not any(\"[DisabledFunction][Exit]\" in msg for msg in log_messages), \"Exit should not be logged when disabled\"\n # Coherence:OK should still be logged (internal tracking)\n assert any(\"[DisabledFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Coherence should still be logged\"\n# [/DEF:test_enable_belief_state_flag:Function]\n\n\n# [DEF:test_belief_scope_missing_anchor:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test @PRE condition: anchor_id must be provided\ndef test_belief_scope_missing_anchor():\n \"\"\"Test that belief_scope enforces anchor_id to be provided.\"\"\"\n import pytest\n from src.core.logger import belief_scope\n with pytest.raises(TypeError):\n # Missing required positional argument 'anchor_id'\n with belief_scope():\n pass\n# [/DEF:test_belief_scope_missing_anchor:Function]\n\n# [DEF:test_configure_logger_post_conditions:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated.\ndef test_configure_logger_post_conditions(tmp_path):\n \"\"\"Test that configure_logger satisfies all @POST conditions.\"\"\"\n import logging\n from logging.handlers import RotatingFileHandler\n from src.core.config_models import LoggingConfig\n from src.core.logger import configure_logger, logger, BeliefFormatter, get_task_log_level\n import src.core.logger as logger_module\n\n log_file = tmp_path / \"test.log\"\n config = LoggingConfig(\n level=\"WARNING\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False,\n file_path=str(log_file)\n )\n \n configure_logger(config)\n \n # 1. Logger level is updated\n assert logger.level == logging.WARNING\n \n # 2. Handlers are updated (file handler removed old ones, added new one)\n file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n assert len(file_handlers) == 1\n import pathlib\n assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve()\n \n # 3. Formatter is set to BeliefFormatter\n for handler in logger.handlers:\n assert isinstance(handler.formatter, BeliefFormatter)\n \n # 4. Global states\n assert getattr(logger_module, '_enable_belief_state') is False\n assert get_task_log_level() == \"DEBUG\"\n# [/DEF:test_configure_logger_post_conditions:Function]\n\n# [/DEF:test_logger:Module]\n" + "body": "# [DEF:test_logger:Module]\n# @COMPLEXITY: 3\n# @PURPOSE: Unit tests for logger module — CoT JSON format markers.\n# @LAYER: Infra\n# @RELATION: VERIFIES -> src.core.logger\n\nimport sys\nfrom pathlib import Path\n\n# Add src to path\nsys.path.append(str(Path(__file__).parent.parent.parent.parent / \"src\"))\n\nimport pytest\nimport logging\nimport json\nfrom src.core.logger import (\n belief_scope,\n logger,\n configure_logger,\n get_task_log_level,\n should_log_task_level,\n CotJsonFormatter,\n)\nfrom src.core.config_models import LoggingConfig\n\n\n@pytest.fixture(autouse=True)\ndef reset_logger_state():\n \"\"\"Reset logger state before each test to avoid cross-test contamination.\"\"\"\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n # Also reset the logger level for caplog to work correctly\n logging.getLogger(\"superset_tools_app\").setLevel(logging.DEBUG)\n yield\n # Reset after test too\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n\n# [DEF:test_belief_scope_logs_reason_reflect_at_debug:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain REASON (entry) and REFLECT (coherence) markers.\ndef test_belief_scope_logs_reason_reflect_at_debug(caplog):\n \"\"\"Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"TestFunction\"):\n logger.info(\"Doing something important\")\n\n # Check that the records contain the expected markers\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction'\n ]\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction'\n ]\n info_records = [\n r for r in caplog.records\n if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()\n ]\n\n assert len(reason_records) >= 1, \"REASON marker not found for TestFunction\"\n assert len(reflect_records) >= 1, \"REFLECT marker not found for TestFunction\"\n assert len(info_records) >= 1, \"INFO log 'Doing something important' not found\"\n\n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_logs_reason_reflect_at_debug:Function]\n\n\n# [DEF:test_belief_scope_error_handling:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs EXPLORE marker on exception.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain EXPLORE marker with error info.\ndef test_belief_scope_error_handling(caplog):\n \"\"\"Test that belief_scope logs EXPLORE marker on exception.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with pytest.raises(ValueError):\n with belief_scope(\"FailingFunction\"):\n raise ValueError(\"Something went wrong\")\n\n # Check that an EXPLORE marker was emitted\n explore_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'EXPLORE'\n and getattr(r, 'src', None) == 'FailingFunction'\n ]\n\n assert len(explore_records) >= 1, \"EXPLORE marker not found for FailingFunction\"\n assert 'Something went wrong' in explore_records[0].getMessage() or \\\n 'Something went wrong' in str(getattr(explore_records[0], 'error', ''))\n\n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_error_handling:Function]\n\n\n# [DEF:test_belief_scope_success_coherence:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs REFLECT marker on success.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain REFLECT marker.\ndef test_belief_scope_success_coherence(caplog):\n \"\"\"Test that belief_scope logs REFLECT marker on success.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"SuccessFunction\"):\n pass\n\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT'\n and getattr(r, 'src', None) == 'SuccessFunction'\n ]\n\n assert len(reflect_records) >= 1, \"REFLECT marker not found for SuccessFunction\"\n assert 'Coherence OK' in reflect_records[0].getMessage() or \\\n getattr(reflect_records[0], 'intent', '') == 'Coherence OK'\n\n\n# [/DEF:test_belief_scope_success_coherence:Function]\n\n\n# [DEF:test_belief_scope_reason_not_visible_at_info:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.\n# @PRE: belief_scope is available. caplog fixture is used.\n# @POST: REASON/REFLECT markers are not captured at INFO level.\ndef test_belief_scope_reason_not_visible_at_info(caplog):\n \"\"\"Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.\"\"\"\n caplog.set_level(\"INFO\")\n\n with belief_scope(\"InfoLevelFunction\"):\n logger.info(\"Doing something important\")\n\n # The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction'\n ]\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction'\n ]\n\n assert len(reason_records) == 0, \"REASON marker should not be visible at INFO\"\n assert len(reflect_records) == 0, \"REFLECT marker should not be visible at INFO\"\n\n # But the INFO-level message should be visible\n info_records = [\n r for r in caplog.records\n if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()\n ]\n assert len(info_records) >= 1, \"INFO log 'Doing something important' should be visible\"\n# [/DEF:test_belief_scope_reason_not_visible_at_info:Function]\n\n\n# [DEF:test_task_log_level_default:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that default task log level is INFO.\n# @PRE: None.\n# @POST: Default level is INFO.\ndef test_task_log_level_default():\n \"\"\"Test that default task log level is INFO (after reset fixture).\"\"\"\n level = get_task_log_level()\n assert level == \"INFO\"\n# [/DEF:test_task_log_level_default:Function]\n\n\n# [DEF:test_should_log_task_level:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that should_log_task_level correctly filters log levels.\n# @PRE: None.\n# @POST: Filtering works correctly for all level combinations.\ndef test_should_log_task_level():\n \"\"\"Test that should_log_task_level correctly filters log levels.\"\"\"\n # Default level is INFO\n assert should_log_task_level(\"ERROR\") is True, \"ERROR should be logged at INFO threshold\"\n assert should_log_task_level(\"WARNING\") is True, \"WARNING should be logged at INFO threshold\"\n assert should_log_task_level(\"INFO\") is True, \"INFO should be logged at INFO threshold\"\n assert should_log_task_level(\"DEBUG\") is False, \"DEBUG should NOT be logged at INFO threshold\"\n# [/DEF:test_should_log_task_level:Function]\n\n\n# [DEF:test_configure_logger_task_log_level:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that configure_logger updates task_log_level.\n# @PRE: LoggingConfig is available.\n# @POST: task_log_level is updated correctly.\ndef test_configure_logger_task_log_level():\n \"\"\"Test that configure_logger updates task_log_level.\"\"\"\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n assert get_task_log_level() == \"DEBUG\", \"task_log_level should be DEBUG\"\n assert should_log_task_level(\"DEBUG\") is True, \"DEBUG should be logged at DEBUG threshold\"\n# [/DEF:test_configure_logger_task_log_level:Function]\n\n\n# [DEF:test_enable_belief_state_flag:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that enable_belief_state flag controls belief_scope entry logging.\n# @PRE: LoggingConfig is available. caplog fixture is used.\n# @POST: REASON entry marker is suppressed when disabled; REFLECT coherence still logged.\ndef test_enable_belief_state_flag(caplog):\n \"\"\"Test that enable_belief_state flag controls belief_scope REASON entry logging.\"\"\"\n # Disable belief state\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"DisabledFunction\"):\n logger.info(\"Doing something\")\n\n # REASON entry marker should NOT be logged when disabled\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction'\n ]\n assert len(reason_records) == 0, \"REASON entry should not be logged when disabled\"\n\n # REFLECT coherence marker should still be logged (internal tracking)\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction'\n ]\n assert len(reflect_records) >= 1, \"REFLECT coherence should still be logged\"\n# [/DEF:test_enable_belief_state_flag:Function]\n\n\n# [DEF:test_belief_scope_missing_anchor:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test @PRE condition: anchor_id must be provided\ndef test_belief_scope_missing_anchor():\n \"\"\"Test that belief_scope enforces anchor_id to be provided.\"\"\"\n import pytest\n from src.core.logger import belief_scope\n with pytest.raises(TypeError):\n # Missing required positional argument 'anchor_id'\n with belief_scope():\n pass\n# [/DEF:test_belief_scope_missing_anchor:Function]\n\n# [DEF:test_configure_logger_post_conditions:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated.\ndef test_configure_logger_post_conditions(tmp_path):\n \"\"\"Test that configure_logger satisfies all @POST conditions.\"\"\"\n import logging\n from logging.handlers import RotatingFileHandler\n from src.core.config_models import LoggingConfig\n from src.core.logger import configure_logger, logger, CotJsonFormatter, get_task_log_level\n import src.core.logger as logger_module\n\n log_file = tmp_path / \"test.log\"\n config = LoggingConfig(\n level=\"WARNING\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False,\n file_path=str(log_file)\n )\n\n configure_logger(config)\n\n # 1. Logger level is updated\n assert logger.level == logging.WARNING\n\n # 2. Handlers are updated (file handler removed old ones, added new one)\n file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n assert len(file_handlers) == 1\n import pathlib\n assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve()\n\n # 3. Formatter is set to CotJsonFormatter\n for handler in logger.handlers:\n assert isinstance(handler.formatter, CotJsonFormatter)\n\n # 4. Global states\n assert getattr(logger_module, '_enable_belief_state') is False\n assert get_task_log_level() == \"DEBUG\"\n# [/DEF:test_configure_logger_post_conditions:Function]\n\n# [DEF:test_cot_json_formatter_output:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that CotJsonFormatter produces valid JSON with expected fields.\ndef test_cot_json_formatter_output():\n \"\"\"Test that CotJsonFormatter produces valid JSON with expected fields.\"\"\"\n import json\n from datetime import datetime, timezone\n\n formatter = CotJsonFormatter()\n\n # Create a LogRecord with structured extra data\n record = logging.LogRecord(\n name=\"test.module\",\n level=logging.INFO,\n pathname=\"/fake/path.py\",\n lineno=42,\n msg=\"Test action\",\n args=(),\n exc_info=None,\n )\n record.marker = \"REASON\"\n record.intent = \"Test action\"\n record.src = \"TestModule\"\n record.payload = {\"key\": \"value\"}\n\n output = formatter.format(record)\n parsed = json.loads(output)\n\n assert parsed[\"level\"] == \"INFO\"\n assert parsed[\"marker\"] == \"REASON\"\n assert parsed[\"intent\"] == \"Test action\"\n assert parsed[\"src\"] == \"TestModule\"\n assert parsed[\"payload\"] == {\"key\": \"value\"}\n assert \"ts\" in parsed\n assert \"trace_id\" in parsed\n# [/DEF:test_cot_json_formatter_output:Function]\n\n# [DEF:test_cot_json_formatter_plain_message:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that CotJsonFormatter wraps plain messages (no extra) with default marker.\ndef test_cot_json_formatter_plain_message():\n \"\"\"Test that CotJsonFormatter wraps plain messages with default marker.\"\"\"\n import json\n\n formatter = CotJsonFormatter()\n\n # Create a LogRecord WITHOUT extra data (plain message)\n record = logging.LogRecord(\n name=\"test.module\",\n level=logging.INFO,\n pathname=\"/fake/path.py\",\n lineno=42,\n msg=\"Plain info message\",\n args=(),\n exc_info=None,\n )\n\n output = formatter.format(record)\n parsed = json.loads(output)\n\n assert parsed[\"level\"] == \"INFO\"\n assert parsed[\"marker\"] == \"REASON\" # default for plain messages\n assert parsed[\"intent\"] == \"Plain info message\"\n assert parsed[\"src\"] == \"test.module\"\n assert \"ts\" in parsed\n assert \"trace_id\" in parsed\n# [/DEF:test_cot_json_formatter_plain_message:Function]\n\n# [/DEF:test_logger:Module]\n" }, { - "contract_id": "test_belief_scope_logs_entry_action_exit_at_debug", + "contract_id": "test_belief_scope_logs_reason_reflect_at_debug", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 46, - "end_line": 76, + "start_line": 48, + "end_line": 89, "tier": "TIER_1", "complexity": 2, "metadata": { - "POST": "Logs are verified to contain Entry, Action, and Exit tags at DEBUG level.", + "POST": "Logs are verified to contain REASON (entry) and REFLECT (coherence) markers.", "PRE": "belief_scope is available. caplog fixture is used. Logger configured to DEBUG.", - "PURPOSE": "Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level." + "PURPOSE": "Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level." }, "relations": [ { - "source_id": "test_belief_scope_logs_entry_action_exit_at_debug", + "source_id": "test_belief_scope_logs_reason_reflect_at_debug", "relation_type": "BINDS_TO", "target_id": "test_logger", "target_ref": "test_logger" @@ -35199,20 +35602,20 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level.\ndef test_belief_scope_logs_entry_action_exit_at_debug(caplog):\n \"\"\"Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"TestFunction\"):\n logger.info(\"Doing something important\")\n\n # Check that the logs contain the expected patterns\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[TestFunction][Entry]\" in msg for msg in log_messages), \"Entry log not found\"\n assert any(\"[TestFunction][Action] Doing something important\" in msg for msg in log_messages), \"Action log not found\"\n assert any(\"[TestFunction][Exit]\" in msg for msg in log_messages), \"Exit log not found\"\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]\n" + "body": "# [DEF:test_belief_scope_logs_reason_reflect_at_debug:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain REASON (entry) and REFLECT (coherence) markers.\ndef test_belief_scope_logs_reason_reflect_at_debug(caplog):\n \"\"\"Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"TestFunction\"):\n logger.info(\"Doing something important\")\n\n # Check that the records contain the expected markers\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction'\n ]\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction'\n ]\n info_records = [\n r for r in caplog.records\n if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()\n ]\n\n assert len(reason_records) >= 1, \"REASON marker not found for TestFunction\"\n assert len(reflect_records) >= 1, \"REFLECT marker not found for TestFunction\"\n assert len(info_records) >= 1, \"INFO log 'Doing something important' not found\"\n\n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_logs_reason_reflect_at_debug:Function]\n" }, { "contract_id": "test_belief_scope_error_handling", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 79, - "end_line": 109, + "start_line": 92, + "end_line": 127, "tier": "TIER_1", "complexity": 2, "metadata": { - "POST": "Logs are verified to contain Coherence:Failed tag.", + "POST": "Logs are verified to contain EXPLORE marker with error info.", "PRE": "belief_scope is available. caplog fixture is used. Logger configured to DEBUG.", - "PURPOSE": "Test that belief_scope logs Coherence:Failed on exception." + "PURPOSE": "Test that belief_scope logs EXPLORE marker on exception." }, "relations": [ { @@ -35252,20 +35655,20 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_belief_scope_error_handling:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Coherence:Failed tag.\ndef test_belief_scope_error_handling(caplog):\n \"\"\"Test that belief_scope logs Coherence:Failed on exception.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with pytest.raises(ValueError):\n with belief_scope(\"FailingFunction\"):\n raise ValueError(\"Something went wrong\")\n\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[FailingFunction][Entry]\" in msg for msg in log_messages), \"Entry log not found\"\n assert any(\"[FailingFunction][COHERENCE:FAILED]\" in msg for msg in log_messages), \"Failed coherence log not found\"\n # Exit should not be logged on failure\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_error_handling:Function]\n" + "body": "# [DEF:test_belief_scope_error_handling:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs EXPLORE marker on exception.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain EXPLORE marker with error info.\ndef test_belief_scope_error_handling(caplog):\n \"\"\"Test that belief_scope logs EXPLORE marker on exception.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with pytest.raises(ValueError):\n with belief_scope(\"FailingFunction\"):\n raise ValueError(\"Something went wrong\")\n\n # Check that an EXPLORE marker was emitted\n explore_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'EXPLORE'\n and getattr(r, 'src', None) == 'FailingFunction'\n ]\n\n assert len(explore_records) >= 1, \"EXPLORE marker not found for FailingFunction\"\n assert 'Something went wrong' in explore_records[0].getMessage() or \\\n 'Something went wrong' in str(getattr(explore_records[0], 'error', ''))\n\n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_error_handling:Function]\n" }, { "contract_id": "test_belief_scope_success_coherence", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 112, - "end_line": 137, + "start_line": 130, + "end_line": 161, "tier": "TIER_1", "complexity": 2, "metadata": { - "POST": "Logs are verified to contain Coherence:OK tag.", + "POST": "Logs are verified to contain REFLECT marker.", "PRE": "belief_scope is available. caplog fixture is used. Logger configured to DEBUG.", - "PURPOSE": "Test that belief_scope logs Coherence:OK on success." + "PURPOSE": "Test that belief_scope logs REFLECT marker on success." }, "relations": [ { @@ -35305,24 +35708,24 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_belief_scope_success_coherence:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs Coherence:OK on success.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Coherence:OK tag.\ndef test_belief_scope_success_coherence(caplog):\n \"\"\"Test that belief_scope logs Coherence:OK on success.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"SuccessFunction\"):\n pass\n\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[SuccessFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Success coherence log not found\"\n \n\n# [/DEF:test_belief_scope_success_coherence:Function]\n" + "body": "# [DEF:test_belief_scope_success_coherence:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope logs REFLECT marker on success.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain REFLECT marker.\ndef test_belief_scope_success_coherence(caplog):\n \"\"\"Test that belief_scope logs REFLECT marker on success.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"SuccessFunction\"):\n pass\n\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT'\n and getattr(r, 'src', None) == 'SuccessFunction'\n ]\n\n assert len(reflect_records) >= 1, \"REFLECT marker not found for SuccessFunction\"\n assert 'Coherence OK' in reflect_records[0].getMessage() or \\\n getattr(reflect_records[0], 'intent', '') == 'Coherence OK'\n\n\n# [/DEF:test_belief_scope_success_coherence:Function]\n" }, { - "contract_id": "test_belief_scope_not_visible_at_info", + "contract_id": "test_belief_scope_reason_not_visible_at_info", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 140, - "end_line": 160, + "start_line": 164, + "end_line": 195, "tier": "TIER_1", "complexity": 2, "metadata": { - "POST": "Entry/Exit/Coherence logs are not captured at INFO level.", + "POST": "REASON/REFLECT markers are not captured at INFO level.", "PRE": "belief_scope is available. caplog fixture is used.", - "PURPOSE": "Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level." + "PURPOSE": "Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level." }, "relations": [ { - "source_id": "test_belief_scope_not_visible_at_info", + "source_id": "test_belief_scope_reason_not_visible_at_info", "relation_type": "BINDS_TO", "target_id": "test_logger", "target_ref": "test_logger" @@ -35358,14 +35761,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_belief_scope_not_visible_at_info:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.\n# @PRE: belief_scope is available. caplog fixture is used.\n# @POST: Entry/Exit/Coherence logs are not captured at INFO level.\ndef test_belief_scope_not_visible_at_info(caplog):\n \"\"\"Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.\"\"\"\n caplog.set_level(\"INFO\")\n\n with belief_scope(\"InfoLevelFunction\"):\n logger.info(\"Doing something important\")\n\n log_messages = [record.message for record in caplog.records]\n\n # Action log should be visible\n assert any(\"[InfoLevelFunction][Action] Doing something important\" in msg for msg in log_messages), \"Action log not found\"\n # Entry/Exit/Coherence should NOT be visible at INFO level\n assert not any(\"[InfoLevelFunction][Entry]\" in msg for msg in log_messages), \"Entry log should not be visible at INFO\"\n assert not any(\"[InfoLevelFunction][Exit]\" in msg for msg in log_messages), \"Exit log should not be visible at INFO\"\n assert not any(\"[InfoLevelFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Coherence log should not be visible at INFO\"\n# [/DEF:test_belief_scope_not_visible_at_info:Function]\n" + "body": "# [DEF:test_belief_scope_reason_not_visible_at_info:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.\n# @PRE: belief_scope is available. caplog fixture is used.\n# @POST: REASON/REFLECT markers are not captured at INFO level.\ndef test_belief_scope_reason_not_visible_at_info(caplog):\n \"\"\"Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.\"\"\"\n caplog.set_level(\"INFO\")\n\n with belief_scope(\"InfoLevelFunction\"):\n logger.info(\"Doing something important\")\n\n # The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction'\n ]\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction'\n ]\n\n assert len(reason_records) == 0, \"REASON marker should not be visible at INFO\"\n assert len(reflect_records) == 0, \"REFLECT marker should not be visible at INFO\"\n\n # But the INFO-level message should be visible\n info_records = [\n r for r in caplog.records\n if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()\n ]\n assert len(info_records) >= 1, \"INFO log 'Doing something important' should be visible\"\n# [/DEF:test_belief_scope_reason_not_visible_at_info:Function]\n" }, { "contract_id": "test_task_log_level_default", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 163, - "end_line": 172, + "start_line": 198, + "end_line": 207, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -35417,8 +35820,8 @@ "contract_id": "test_should_log_task_level", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 175, - "end_line": 187, + "start_line": 210, + "end_line": 222, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -35470,8 +35873,8 @@ "contract_id": "test_configure_logger_task_log_level", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 190, - "end_line": 206, + "start_line": 225, + "end_line": 241, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -35517,20 +35920,20 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_configure_logger_task_log_level:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that configure_logger updates task_log_level.\n# @PRE: LoggingConfig is available.\n# @POST: task_log_level is updated correctly.\ndef test_configure_logger_task_log_level():\n \"\"\"Test that configure_logger updates task_log_level.\"\"\"\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n assert get_task_log_level() == \"DEBUG\", \"task_log_level should be DEBUG\"\n assert should_log_task_level(\"DEBUG\") is True, \"DEBUG should be logged at DEBUG threshold\"\n# [/DEF:test_configure_logger_task_log_level:Function]\n" + "body": "# [DEF:test_configure_logger_task_log_level:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that configure_logger updates task_log_level.\n# @PRE: LoggingConfig is available.\n# @POST: task_log_level is updated correctly.\ndef test_configure_logger_task_log_level():\n \"\"\"Test that configure_logger updates task_log_level.\"\"\"\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n assert get_task_log_level() == \"DEBUG\", \"task_log_level should be DEBUG\"\n assert should_log_task_level(\"DEBUG\") is True, \"DEBUG should be logged at DEBUG threshold\"\n# [/DEF:test_configure_logger_task_log_level:Function]\n" }, { "contract_id": "test_enable_belief_state_flag", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 209, - "end_line": 236, + "start_line": 244, + "end_line": 277, "tier": "TIER_1", "complexity": 2, "metadata": { - "POST": "belief_scope logs are controlled by the flag.", + "POST": "REASON entry marker is suppressed when disabled; REFLECT coherence still logged.", "PRE": "LoggingConfig is available. caplog fixture is used.", - "PURPOSE": "Test that enable_belief_state flag controls belief_scope logging." + "PURPOSE": "Test that enable_belief_state flag controls belief_scope entry logging." }, "relations": [ { @@ -35570,14 +35973,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_enable_belief_state_flag:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging.\n# @PRE: LoggingConfig is available. caplog fixture is used.\n# @POST: belief_scope logs are controlled by the flag.\ndef test_enable_belief_state_flag(caplog):\n \"\"\"Test that enable_belief_state flag controls belief_scope logging.\"\"\"\n # Disable belief state\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n \n with belief_scope(\"DisabledFunction\"):\n logger.info(\"Doing something\")\n \n log_messages = [record.message for record in caplog.records]\n \n # Entry and Exit should NOT be logged when disabled\n assert not any(\"[DisabledFunction][Entry]\" in msg for msg in log_messages), \"Entry should not be logged when disabled\"\n assert not any(\"[DisabledFunction][Exit]\" in msg for msg in log_messages), \"Exit should not be logged when disabled\"\n # Coherence:OK should still be logged (internal tracking)\n assert any(\"[DisabledFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Coherence should still be logged\"\n# [/DEF:test_enable_belief_state_flag:Function]\n" + "body": "# [DEF:test_enable_belief_state_flag:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that enable_belief_state flag controls belief_scope entry logging.\n# @PRE: LoggingConfig is available. caplog fixture is used.\n# @POST: REASON entry marker is suppressed when disabled; REFLECT coherence still logged.\ndef test_enable_belief_state_flag(caplog):\n \"\"\"Test that enable_belief_state flag controls belief_scope REASON entry logging.\"\"\"\n # Disable belief state\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"DisabledFunction\"):\n logger.info(\"Doing something\")\n\n # REASON entry marker should NOT be logged when disabled\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction'\n ]\n assert len(reason_records) == 0, \"REASON entry should not be logged when disabled\"\n\n # REFLECT coherence marker should still be logged (internal tracking)\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction'\n ]\n assert len(reflect_records) >= 1, \"REFLECT coherence should still be logged\"\n# [/DEF:test_enable_belief_state_flag:Function]\n" }, { "contract_id": "test_belief_scope_missing_anchor", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 239, - "end_line": 250, + "start_line": 280, + "end_line": 291, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -35609,8 +36012,8 @@ "contract_id": "test_configure_logger_post_conditions", "contract_type": "Function", "file_path": "backend/src/core/logger/__tests__/test_logger.py", - "start_line": 252, - "end_line": 289, + "start_line": 293, + "end_line": 330, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -35636,14 +36039,80 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:test_configure_logger_post_conditions:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated.\ndef test_configure_logger_post_conditions(tmp_path):\n \"\"\"Test that configure_logger satisfies all @POST conditions.\"\"\"\n import logging\n from logging.handlers import RotatingFileHandler\n from src.core.config_models import LoggingConfig\n from src.core.logger import configure_logger, logger, BeliefFormatter, get_task_log_level\n import src.core.logger as logger_module\n\n log_file = tmp_path / \"test.log\"\n config = LoggingConfig(\n level=\"WARNING\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False,\n file_path=str(log_file)\n )\n \n configure_logger(config)\n \n # 1. Logger level is updated\n assert logger.level == logging.WARNING\n \n # 2. Handlers are updated (file handler removed old ones, added new one)\n file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n assert len(file_handlers) == 1\n import pathlib\n assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve()\n \n # 3. Formatter is set to BeliefFormatter\n for handler in logger.handlers:\n assert isinstance(handler.formatter, BeliefFormatter)\n \n # 4. Global states\n assert getattr(logger_module, '_enable_belief_state') is False\n assert get_task_log_level() == \"DEBUG\"\n# [/DEF:test_configure_logger_post_conditions:Function]\n" + "body": "# [DEF:test_configure_logger_post_conditions:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated.\ndef test_configure_logger_post_conditions(tmp_path):\n \"\"\"Test that configure_logger satisfies all @POST conditions.\"\"\"\n import logging\n from logging.handlers import RotatingFileHandler\n from src.core.config_models import LoggingConfig\n from src.core.logger import configure_logger, logger, CotJsonFormatter, get_task_log_level\n import src.core.logger as logger_module\n\n log_file = tmp_path / \"test.log\"\n config = LoggingConfig(\n level=\"WARNING\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False,\n file_path=str(log_file)\n )\n\n configure_logger(config)\n\n # 1. Logger level is updated\n assert logger.level == logging.WARNING\n\n # 2. Handlers are updated (file handler removed old ones, added new one)\n file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]\n assert len(file_handlers) == 1\n import pathlib\n assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve()\n\n # 3. Formatter is set to CotJsonFormatter\n for handler in logger.handlers:\n assert isinstance(handler.formatter, CotJsonFormatter)\n\n # 4. Global states\n assert getattr(logger_module, '_enable_belief_state') is False\n assert get_task_log_level() == \"DEBUG\"\n# [/DEF:test_configure_logger_post_conditions:Function]\n" + }, + { + "contract_id": "test_cot_json_formatter_output", + "contract_type": "Function", + "file_path": "backend/src/core/logger/__tests__/test_logger.py", + "start_line": 332, + "end_line": 367, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "PURPOSE": "Test that CotJsonFormatter produces valid JSON with expected fields." + }, + "relations": [ + { + "source_id": "test_cot_json_formatter_output", + "relation_type": "BINDS_TO", + "target_id": "test_logger", + "target_ref": "test_logger" + } + ], + "schema_warnings": [ + { + "code": "tag_forbidden_by_complexity", + "tag": "RELATION", + "message": "@RELATION is forbidden for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:test_cot_json_formatter_output:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that CotJsonFormatter produces valid JSON with expected fields.\ndef test_cot_json_formatter_output():\n \"\"\"Test that CotJsonFormatter produces valid JSON with expected fields.\"\"\"\n import json\n from datetime import datetime, timezone\n\n formatter = CotJsonFormatter()\n\n # Create a LogRecord with structured extra data\n record = logging.LogRecord(\n name=\"test.module\",\n level=logging.INFO,\n pathname=\"/fake/path.py\",\n lineno=42,\n msg=\"Test action\",\n args=(),\n exc_info=None,\n )\n record.marker = \"REASON\"\n record.intent = \"Test action\"\n record.src = \"TestModule\"\n record.payload = {\"key\": \"value\"}\n\n output = formatter.format(record)\n parsed = json.loads(output)\n\n assert parsed[\"level\"] == \"INFO\"\n assert parsed[\"marker\"] == \"REASON\"\n assert parsed[\"intent\"] == \"Test action\"\n assert parsed[\"src\"] == \"TestModule\"\n assert parsed[\"payload\"] == {\"key\": \"value\"}\n assert \"ts\" in parsed\n assert \"trace_id\" in parsed\n# [/DEF:test_cot_json_formatter_output:Function]\n" + }, + { + "contract_id": "test_cot_json_formatter_plain_message", + "contract_type": "Function", + "file_path": "backend/src/core/logger/__tests__/test_logger.py", + "start_line": 369, + "end_line": 398, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "PURPOSE": "Test that CotJsonFormatter wraps plain messages (no extra) with default marker." + }, + "relations": [ + { + "source_id": "test_cot_json_formatter_plain_message", + "relation_type": "BINDS_TO", + "target_id": "test_logger", + "target_ref": "test_logger" + } + ], + "schema_warnings": [ + { + "code": "tag_forbidden_by_complexity", + "tag": "RELATION", + "message": "@RELATION is forbidden for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:test_cot_json_formatter_plain_message:Function]\n# @RELATION: BINDS_TO -> test_logger\n# @PURPOSE: Test that CotJsonFormatter wraps plain messages (no extra) with default marker.\ndef test_cot_json_formatter_plain_message():\n \"\"\"Test that CotJsonFormatter wraps plain messages with default marker.\"\"\"\n import json\n\n formatter = CotJsonFormatter()\n\n # Create a LogRecord WITHOUT extra data (plain message)\n record = logging.LogRecord(\n name=\"test.module\",\n level=logging.INFO,\n pathname=\"/fake/path.py\",\n lineno=42,\n msg=\"Plain info message\",\n args=(),\n exc_info=None,\n )\n\n output = formatter.format(record)\n parsed = json.loads(output)\n\n assert parsed[\"level\"] == \"INFO\"\n assert parsed[\"marker\"] == \"REASON\" # default for plain messages\n assert parsed[\"intent\"] == \"Plain info message\"\n assert parsed[\"src\"] == \"test.module\"\n assert \"ts\" in parsed\n assert \"trace_id\" in parsed\n# [/DEF:test_cot_json_formatter_plain_message:Function]\n" }, { "contract_id": "IdMappingServiceModule", "contract_type": "Module", "file_path": "backend/src/core/mapping_service.py", "start_line": 1, - "end_line": 316, + "end_line": 301, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -35701,14 +36170,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:IdMappingServiceModule:Module]\n#\n# @COMPLEXITY: 5\n# @SEMANTICS: mapping, ids, synchronization, environments, cross-filters\n# @PURPOSE: Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)\n# @LAYER: Core\n# @RELATION: DEPENDS_ON -> [MappingModels]\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments.\n# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.\n# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.\n# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]\n# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}\n#\n# @INVARIANT: sync_environment must handle remote API failures gracefully.\n\n# [SECTION: IMPORTS]\nfrom typing import Dict, List, Optional\nfrom datetime import datetime, timezone\nfrom sqlalchemy.orm import Session\nfrom apscheduler.schedulers.background import BackgroundScheduler\nfrom apscheduler.triggers.cron import CronTrigger\nfrom src.models.mapping import ResourceMapping, ResourceType\nfrom src.core.logger import logger, belief_scope\n# [/SECTION]\n\n\n# [DEF:IdMappingService:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.\n# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.\n# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.\n# @RELATION: DEPENDS_ON -> [MappingModels]\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @INVARIANT: self.db remains the authoritative session for all mapping operations.\n# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.\n# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]\n#\n# @TEST_CONTRACT: IdMappingServiceModel ->\n# {\n# required_fields: {db_session: Session},\n# invariants: [\n# \"sync_environment correctly creates or updates ResourceMapping records\",\n# \"get_remote_id returns an integer or None\",\n# \"get_remote_ids_batch returns a dictionary of valid UUIDs to integers\"\n# ]\n# }\n# @TEST_FIXTURE: valid_mapping_service -> {\"db_session\": \"MockSession()\"}\n# @TEST_EDGE: sync_api_failure -> handles exception gracefully\n# @TEST_EDGE: get_remote_id_not_found -> returns None\n# @TEST_EDGE: get_batch_empty_list -> returns empty dict\n# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]\nclass IdMappingService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapping service.\n def __init__(self, db_session: Session):\n self.db = db_session\n self.scheduler = BackgroundScheduler()\n self._sync_job = None\n\n # [/DEF:__init__:Function]\n\n # [DEF:start_scheduler:Function]\n # @PURPOSE: Starts the background scheduler with a given cron string.\n # @PARAM: cron_string (str) - Cron expression for the sync interval.\n # @PARAM: environments (List[str]) - List of environment IDs to sync.\n # @PARAM: superset_client_factory - Function to get a client for an environment.\n def start_scheduler(\n self, cron_string: str, environments: List[str], superset_client_factory\n ):\n with belief_scope(\"IdMappingService.start_scheduler\"):\n if self._sync_job:\n self.scheduler.remove_job(self._sync_job.id)\n logger.info(\n \"[IdMappingService.start_scheduler][Reflect] Removed existing sync job.\"\n )\n\n def sync_all():\n for env_id in environments:\n client = superset_client_factory(env_id)\n if client:\n self.sync_environment(env_id, client)\n\n self._sync_job = self.scheduler.add_job(\n sync_all,\n CronTrigger.from_crontab(cron_string),\n id=\"id_mapping_sync_job\",\n replace_existing=True,\n )\n\n if not self.scheduler.running:\n self.scheduler.start()\n logger.info(\n f\"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}\"\n )\n else:\n logger.info(\n f\"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}\"\n )\n\n # [/DEF:start_scheduler:Function]\n\n # [DEF:sync_environment:Function]\n # @PURPOSE: Fully synchronizes mapping for a specific environment.\n # @PARAM: environment_id (str) - Target environment ID.\n # @PARAM: superset_client - Instance capable of hitting the Superset API.\n # @PRE: environment_id exists in the database.\n # @POST: ResourceMapping records for the environment are created or updated.\n def sync_environment(\n self, environment_id: str, superset_client, incremental: bool = False\n ) -> None:\n \"\"\"\n Polls the Superset APIs for the target environment and updates the local mapping table.\n If incremental=True, only fetches items changed since the max last_synced_at date.\n \"\"\"\n with belief_scope(\"IdMappingService.sync_environment\"):\n logger.info(\n f\"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})\"\n )\n\n # Implementation Note: In a real scenario, superset_client needs to be an instance\n # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/\n # Here we structure the logic according to the spec.\n\n types_to_poll = [\n (ResourceType.CHART, \"chart\", \"slice_name\"),\n (ResourceType.DATASET, \"dataset\", \"table_name\"),\n (\n ResourceType.DASHBOARD,\n \"dashboard\",\n \"slug\",\n ), # Note: dashboard slug or dashboard_title\n ]\n\n total_synced = 0\n total_deleted = 0\n try:\n for res_enum, endpoint, name_field in types_to_poll:\n logger.debug(\n f\"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint\"\n )\n\n # Simulated API Fetch (Would be: superset_client.get(f\"/api/v1/{endpoint}/\")... )\n # This relies on the superset API structure, e.g. { \"result\": [{\"id\": 1, \"uuid\": \"...\", name_field: \"...\"}] }\n # We assume superset_client provides a generic method to fetch all pages.\n\n try:\n since_dttm = None\n if incremental:\n from sqlalchemy.sql import func\n\n max_date = (\n self.db.query(func.max(ResourceMapping.last_synced_at))\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n .scalar()\n )\n\n if max_date:\n # We subtract a bit for safety overlap\n from datetime import timedelta\n\n since_dttm = max_date - timedelta(minutes=5)\n logger.debug(\n f\"[IdMappingService.sync_environment] Incremental sync since {since_dttm}\"\n )\n\n resources = superset_client.get_all_resources(\n endpoint, since_dttm=since_dttm\n )\n\n # Track which UUIDs we see in this sync cycle\n synced_uuids = set()\n\n for res in resources:\n res_uuid = res.get(\"uuid\")\n raw_id = res.get(\"id\")\n res_name = res.get(name_field)\n\n if not res_uuid or raw_id is None:\n continue\n\n synced_uuids.add(res_uuid)\n res_id = str(raw_id) # Store as string\n\n # Upsert Logic\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n )\n .first()\n )\n\n if mapping:\n mapping.remote_integer_id = res_id\n mapping.resource_name = res_name\n mapping.last_synced_at = datetime.now(timezone.utc)\n else:\n new_mapping = ResourceMapping(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n remote_integer_id=res_id,\n resource_name=res_name,\n last_synced_at=datetime.now(timezone.utc),\n )\n self.db.add(new_mapping)\n\n total_synced += 1\n\n # Delete stale mappings: rows for this env+type whose UUID\n # was NOT returned by the API (resource was deleted remotely)\n # We only do this on full syncs, because incremental syncs don't return all UUIDs\n if not incremental:\n stale_query = self.db.query(ResourceMapping).filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n if synced_uuids:\n stale_query = stale_query.filter(\n ResourceMapping.uuid.notin_(synced_uuids)\n )\n deleted = stale_query.delete(synchronize_session=\"fetch\")\n if deleted:\n total_deleted += deleted\n logger.info(\n f\"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}\"\n )\n\n except Exception as loop_e:\n logger.error(\n f\"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}\"\n )\n # Continue to next resource type instead of blowing up the whole sync\n\n self.db.commit()\n logger.info(\n f\"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items.\"\n )\n\n except Exception as e:\n self.db.rollback()\n logger.error(\n f\"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}\"\n )\n raise\n\n # [/DEF:sync_environment:Function]\n\n # [DEF:get_remote_id:Function]\n # @PURPOSE: Retrieves the remote integer ID for a given universal UUID.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuid (str)\n # @RETURN: Optional[int]\n def get_remote_id(\n self, environment_id: str, resource_type: ResourceType, uuid: str\n ) -> Optional[int]:\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id, resource_type=resource_type, uuid=uuid\n )\n .first()\n )\n\n if mapping:\n try:\n return int(mapping.remote_integer_id)\n except ValueError:\n return None\n return None\n\n # [/DEF:get_remote_id:Function]\n\n # [DEF:get_remote_ids_batch:Function]\n # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuids (List[str])\n # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID\n def get_remote_ids_batch(\n self, environment_id: str, resource_type: ResourceType, uuids: List[str]\n ) -> Dict[str, int]:\n if not uuids:\n return {}\n\n mappings = (\n self.db.query(ResourceMapping)\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == resource_type,\n ResourceMapping.uuid.in_(uuids),\n )\n .all()\n )\n\n result = {}\n for m in mappings:\n try:\n result[m.uuid] = int(m.remote_integer_id)\n except ValueError:\n pass\n\n return result\n\n # [/DEF:get_remote_ids_batch:Function]\n\n\n# [/DEF:IdMappingService:Class]\n# [/DEF:IdMappingServiceModule:Module]\n" + "body": "# [DEF:IdMappingServiceModule:Module]\n#\n# @COMPLEXITY: 5\n# @SEMANTICS: mapping, ids, synchronization, environments, cross-filters\n# @PURPOSE: Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)\n# @LAYER: Core\n# @RELATION: DEPENDS_ON -> [MappingModels]\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments.\n# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.\n# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.\n# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]\n# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}\n#\n# @INVARIANT: sync_environment must handle remote API failures gracefully.\n\n# [SECTION: IMPORTS]\nfrom typing import Dict, List, Optional\nfrom datetime import datetime, timezone\nfrom sqlalchemy.orm import Session\nfrom apscheduler.schedulers.background import BackgroundScheduler\nfrom apscheduler.triggers.cron import CronTrigger\nfrom src.models.mapping import ResourceMapping, ResourceType\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import logger, belief_scope\n\nlog = MarkerLogger(\"IdMapping\")\n# [/SECTION]\n\n\n# [DEF:IdMappingService:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.\n# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.\n# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.\n# @RELATION: DEPENDS_ON -> [MappingModels]\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @INVARIANT: self.db remains the authoritative session for all mapping operations.\n# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.\n# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]\n#\n# @TEST_CONTRACT: IdMappingServiceModel ->\n# {\n# required_fields: {db_session: Session},\n# invariants: [\n# \"sync_environment correctly creates or updates ResourceMapping records\",\n# \"get_remote_id returns an integer or None\",\n# \"get_remote_ids_batch returns a dictionary of valid UUIDs to integers\"\n# ]\n# }\n# @TEST_FIXTURE: valid_mapping_service -> {\"db_session\": \"MockSession()\"}\n# @TEST_EDGE: sync_api_failure -> handles exception gracefully\n# @TEST_EDGE: get_remote_id_not_found -> returns None\n# @TEST_EDGE: get_batch_empty_list -> returns empty dict\n# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]\nclass IdMappingService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapping service.\n def __init__(self, db_session: Session):\n self.db = db_session\n self.scheduler = BackgroundScheduler()\n self._sync_job = None\n\n # [/DEF:__init__:Function]\n\n # [DEF:start_scheduler:Function]\n # @PURPOSE: Starts the background scheduler with a given cron string.\n # @PARAM: cron_string (str) - Cron expression for the sync interval.\n # @PARAM: environments (List[str]) - List of environment IDs to sync.\n # @PARAM: superset_client_factory - Function to get a client for an environment.\n def start_scheduler(\n self, cron_string: str, environments: List[str], superset_client_factory\n ):\n with belief_scope(\"IdMappingService.start_scheduler\"):\n if self._sync_job:\n self.scheduler.remove_job(self._sync_job.id)\n log.reflect(\"Removed existing sync job.\")\n\n def sync_all():\n for env_id in environments:\n client = superset_client_factory(env_id)\n if client:\n self.sync_environment(env_id, client)\n\n self._sync_job = self.scheduler.add_job(\n sync_all,\n CronTrigger.from_crontab(cron_string),\n id=\"id_mapping_sync_job\",\n replace_existing=True,\n )\n\n if not self.scheduler.running:\n self.scheduler.start()\n log.reason(f\"Started background scheduler with cron: {cron_string}\")\n else:\n log.reason(f\"Updated background scheduler with cron: {cron_string}\")\n\n # [/DEF:start_scheduler:Function]\n\n # [DEF:sync_environment:Function]\n # @PURPOSE: Fully synchronizes mapping for a specific environment.\n # @PARAM: environment_id (str) - Target environment ID.\n # @PARAM: superset_client - Instance capable of hitting the Superset API.\n # @PRE: environment_id exists in the database.\n # @POST: ResourceMapping records for the environment are created or updated.\n def sync_environment(\n self, environment_id: str, superset_client, incremental: bool = False\n ) -> None:\n \"\"\"\n Polls the Superset APIs for the target environment and updates the local mapping table.\n If incremental=True, only fetches items changed since the max last_synced_at date.\n \"\"\"\n with belief_scope(\"IdMappingService.sync_environment\"):\n log.reason(f\"Starting sync for environment {environment_id} (incremental={incremental})\")\n\n # Implementation Note: In a real scenario, superset_client needs to be an instance\n # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/\n # Here we structure the logic according to the spec.\n\n types_to_poll = [\n (ResourceType.CHART, \"chart\", \"slice_name\"),\n (ResourceType.DATASET, \"dataset\", \"table_name\"),\n (\n ResourceType.DASHBOARD,\n \"dashboard\",\n \"slug\",\n ), # Note: dashboard slug or dashboard_title\n ]\n\n total_synced = 0\n total_deleted = 0\n try:\n for res_enum, endpoint, name_field in types_to_poll:\n log.reason(f\"Polling {endpoint} endpoint\")\n\n # Simulated API Fetch (Would be: superset_client.get(f\"/api/v1/{endpoint}/\")... )\n # This relies on the superset API structure, e.g. { \"result\": [{\"id\": 1, \"uuid\": \"...\", name_field: \"...\"}] }\n # We assume superset_client provides a generic method to fetch all pages.\n\n try:\n since_dttm = None\n if incremental:\n from sqlalchemy.sql import func\n\n max_date = (\n self.db.query(func.max(ResourceMapping.last_synced_at))\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n .scalar()\n )\n\n if max_date:\n # We subtract a bit for safety overlap\n from datetime import timedelta\n\n since_dttm = max_date - timedelta(minutes=5)\n log.reason(\n f\"Incremental sync since {since_dttm}\"\n )\n\n resources = superset_client.get_all_resources(\n endpoint, since_dttm=since_dttm\n )\n\n # Track which UUIDs we see in this sync cycle\n synced_uuids = set()\n\n for res in resources:\n res_uuid = res.get(\"uuid\")\n raw_id = res.get(\"id\")\n res_name = res.get(name_field)\n\n if not res_uuid or raw_id is None:\n continue\n\n synced_uuids.add(res_uuid)\n res_id = str(raw_id) # Store as string\n\n # Upsert Logic\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n )\n .first()\n )\n\n if mapping:\n mapping.remote_integer_id = res_id\n mapping.resource_name = res_name\n mapping.last_synced_at = datetime.now(timezone.utc)\n else:\n new_mapping = ResourceMapping(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n remote_integer_id=res_id,\n resource_name=res_name,\n last_synced_at=datetime.now(timezone.utc),\n )\n self.db.add(new_mapping)\n\n total_synced += 1\n\n # Delete stale mappings: rows for this env+type whose UUID\n # was NOT returned by the API (resource was deleted remotely)\n # We only do this on full syncs, because incremental syncs don't return all UUIDs\n if not incremental:\n stale_query = self.db.query(ResourceMapping).filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n if synced_uuids:\n stale_query = stale_query.filter(\n ResourceMapping.uuid.notin_(synced_uuids)\n )\n deleted = stale_query.delete(synchronize_session=\"fetch\")\n if deleted:\n total_deleted += deleted\n log.reason(f\"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}\")\n\n except Exception as loop_e:\n log.explore(f\"Error polling {endpoint}\", error=str(loop_e))\n # Continue to next resource type instead of blowing up the whole sync\n\n self.db.commit()\n log.reflect(f\"Successfully synced {total_synced} items and deleted {total_deleted} stale items.\")\n\n except Exception as e:\n self.db.rollback()\n log.explore(\"Critical sync failure\", error=str(e))\n raise\n\n # [/DEF:sync_environment:Function]\n\n # [DEF:get_remote_id:Function]\n # @PURPOSE: Retrieves the remote integer ID for a given universal UUID.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuid (str)\n # @RETURN: Optional[int]\n def get_remote_id(\n self, environment_id: str, resource_type: ResourceType, uuid: str\n ) -> Optional[int]:\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id, resource_type=resource_type, uuid=uuid\n )\n .first()\n )\n\n if mapping:\n try:\n return int(mapping.remote_integer_id)\n except ValueError:\n return None\n return None\n\n # [/DEF:get_remote_id:Function]\n\n # [DEF:get_remote_ids_batch:Function]\n # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuids (List[str])\n # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID\n def get_remote_ids_batch(\n self, environment_id: str, resource_type: ResourceType, uuids: List[str]\n ) -> Dict[str, int]:\n if not uuids:\n return {}\n\n mappings = (\n self.db.query(ResourceMapping)\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == resource_type,\n ResourceMapping.uuid.in_(uuids),\n )\n .all()\n )\n\n result = {}\n for m in mappings:\n try:\n result[m.uuid] = int(m.remote_integer_id)\n except ValueError:\n pass\n\n return result\n\n # [/DEF:get_remote_ids_batch:Function]\n\n\n# [/DEF:IdMappingService:Class]\n# [/DEF:IdMappingServiceModule:Module]\n" }, { "contract_id": "IdMappingService", "contract_type": "Class", "file_path": "backend/src/core/mapping_service.py", - "start_line": 28, - "end_line": 315, + "start_line": 31, + "end_line": 300, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -35759,14 +36228,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:IdMappingService:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.\n# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.\n# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.\n# @RELATION: DEPENDS_ON -> [MappingModels]\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @INVARIANT: self.db remains the authoritative session for all mapping operations.\n# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.\n# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]\n#\n# @TEST_CONTRACT: IdMappingServiceModel ->\n# {\n# required_fields: {db_session: Session},\n# invariants: [\n# \"sync_environment correctly creates or updates ResourceMapping records\",\n# \"get_remote_id returns an integer or None\",\n# \"get_remote_ids_batch returns a dictionary of valid UUIDs to integers\"\n# ]\n# }\n# @TEST_FIXTURE: valid_mapping_service -> {\"db_session\": \"MockSession()\"}\n# @TEST_EDGE: sync_api_failure -> handles exception gracefully\n# @TEST_EDGE: get_remote_id_not_found -> returns None\n# @TEST_EDGE: get_batch_empty_list -> returns empty dict\n# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]\nclass IdMappingService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapping service.\n def __init__(self, db_session: Session):\n self.db = db_session\n self.scheduler = BackgroundScheduler()\n self._sync_job = None\n\n # [/DEF:__init__:Function]\n\n # [DEF:start_scheduler:Function]\n # @PURPOSE: Starts the background scheduler with a given cron string.\n # @PARAM: cron_string (str) - Cron expression for the sync interval.\n # @PARAM: environments (List[str]) - List of environment IDs to sync.\n # @PARAM: superset_client_factory - Function to get a client for an environment.\n def start_scheduler(\n self, cron_string: str, environments: List[str], superset_client_factory\n ):\n with belief_scope(\"IdMappingService.start_scheduler\"):\n if self._sync_job:\n self.scheduler.remove_job(self._sync_job.id)\n logger.info(\n \"[IdMappingService.start_scheduler][Reflect] Removed existing sync job.\"\n )\n\n def sync_all():\n for env_id in environments:\n client = superset_client_factory(env_id)\n if client:\n self.sync_environment(env_id, client)\n\n self._sync_job = self.scheduler.add_job(\n sync_all,\n CronTrigger.from_crontab(cron_string),\n id=\"id_mapping_sync_job\",\n replace_existing=True,\n )\n\n if not self.scheduler.running:\n self.scheduler.start()\n logger.info(\n f\"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}\"\n )\n else:\n logger.info(\n f\"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}\"\n )\n\n # [/DEF:start_scheduler:Function]\n\n # [DEF:sync_environment:Function]\n # @PURPOSE: Fully synchronizes mapping for a specific environment.\n # @PARAM: environment_id (str) - Target environment ID.\n # @PARAM: superset_client - Instance capable of hitting the Superset API.\n # @PRE: environment_id exists in the database.\n # @POST: ResourceMapping records for the environment are created or updated.\n def sync_environment(\n self, environment_id: str, superset_client, incremental: bool = False\n ) -> None:\n \"\"\"\n Polls the Superset APIs for the target environment and updates the local mapping table.\n If incremental=True, only fetches items changed since the max last_synced_at date.\n \"\"\"\n with belief_scope(\"IdMappingService.sync_environment\"):\n logger.info(\n f\"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})\"\n )\n\n # Implementation Note: In a real scenario, superset_client needs to be an instance\n # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/\n # Here we structure the logic according to the spec.\n\n types_to_poll = [\n (ResourceType.CHART, \"chart\", \"slice_name\"),\n (ResourceType.DATASET, \"dataset\", \"table_name\"),\n (\n ResourceType.DASHBOARD,\n \"dashboard\",\n \"slug\",\n ), # Note: dashboard slug or dashboard_title\n ]\n\n total_synced = 0\n total_deleted = 0\n try:\n for res_enum, endpoint, name_field in types_to_poll:\n logger.debug(\n f\"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint\"\n )\n\n # Simulated API Fetch (Would be: superset_client.get(f\"/api/v1/{endpoint}/\")... )\n # This relies on the superset API structure, e.g. { \"result\": [{\"id\": 1, \"uuid\": \"...\", name_field: \"...\"}] }\n # We assume superset_client provides a generic method to fetch all pages.\n\n try:\n since_dttm = None\n if incremental:\n from sqlalchemy.sql import func\n\n max_date = (\n self.db.query(func.max(ResourceMapping.last_synced_at))\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n .scalar()\n )\n\n if max_date:\n # We subtract a bit for safety overlap\n from datetime import timedelta\n\n since_dttm = max_date - timedelta(minutes=5)\n logger.debug(\n f\"[IdMappingService.sync_environment] Incremental sync since {since_dttm}\"\n )\n\n resources = superset_client.get_all_resources(\n endpoint, since_dttm=since_dttm\n )\n\n # Track which UUIDs we see in this sync cycle\n synced_uuids = set()\n\n for res in resources:\n res_uuid = res.get(\"uuid\")\n raw_id = res.get(\"id\")\n res_name = res.get(name_field)\n\n if not res_uuid or raw_id is None:\n continue\n\n synced_uuids.add(res_uuid)\n res_id = str(raw_id) # Store as string\n\n # Upsert Logic\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n )\n .first()\n )\n\n if mapping:\n mapping.remote_integer_id = res_id\n mapping.resource_name = res_name\n mapping.last_synced_at = datetime.now(timezone.utc)\n else:\n new_mapping = ResourceMapping(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n remote_integer_id=res_id,\n resource_name=res_name,\n last_synced_at=datetime.now(timezone.utc),\n )\n self.db.add(new_mapping)\n\n total_synced += 1\n\n # Delete stale mappings: rows for this env+type whose UUID\n # was NOT returned by the API (resource was deleted remotely)\n # We only do this on full syncs, because incremental syncs don't return all UUIDs\n if not incremental:\n stale_query = self.db.query(ResourceMapping).filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n if synced_uuids:\n stale_query = stale_query.filter(\n ResourceMapping.uuid.notin_(synced_uuids)\n )\n deleted = stale_query.delete(synchronize_session=\"fetch\")\n if deleted:\n total_deleted += deleted\n logger.info(\n f\"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}\"\n )\n\n except Exception as loop_e:\n logger.error(\n f\"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}\"\n )\n # Continue to next resource type instead of blowing up the whole sync\n\n self.db.commit()\n logger.info(\n f\"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items.\"\n )\n\n except Exception as e:\n self.db.rollback()\n logger.error(\n f\"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}\"\n )\n raise\n\n # [/DEF:sync_environment:Function]\n\n # [DEF:get_remote_id:Function]\n # @PURPOSE: Retrieves the remote integer ID for a given universal UUID.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuid (str)\n # @RETURN: Optional[int]\n def get_remote_id(\n self, environment_id: str, resource_type: ResourceType, uuid: str\n ) -> Optional[int]:\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id, resource_type=resource_type, uuid=uuid\n )\n .first()\n )\n\n if mapping:\n try:\n return int(mapping.remote_integer_id)\n except ValueError:\n return None\n return None\n\n # [/DEF:get_remote_id:Function]\n\n # [DEF:get_remote_ids_batch:Function]\n # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuids (List[str])\n # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID\n def get_remote_ids_batch(\n self, environment_id: str, resource_type: ResourceType, uuids: List[str]\n ) -> Dict[str, int]:\n if not uuids:\n return {}\n\n mappings = (\n self.db.query(ResourceMapping)\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == resource_type,\n ResourceMapping.uuid.in_(uuids),\n )\n .all()\n )\n\n result = {}\n for m in mappings:\n try:\n result[m.uuid] = int(m.remote_integer_id)\n except ValueError:\n pass\n\n return result\n\n # [/DEF:get_remote_ids_batch:Function]\n\n\n# [/DEF:IdMappingService:Class]\n" + "body": "# [DEF:IdMappingService:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.\n# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.\n# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.\n# @RELATION: DEPENDS_ON -> [MappingModels]\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @INVARIANT: self.db remains the authoritative session for all mapping operations.\n# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.\n# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]\n#\n# @TEST_CONTRACT: IdMappingServiceModel ->\n# {\n# required_fields: {db_session: Session},\n# invariants: [\n# \"sync_environment correctly creates or updates ResourceMapping records\",\n# \"get_remote_id returns an integer or None\",\n# \"get_remote_ids_batch returns a dictionary of valid UUIDs to integers\"\n# ]\n# }\n# @TEST_FIXTURE: valid_mapping_service -> {\"db_session\": \"MockSession()\"}\n# @TEST_EDGE: sync_api_failure -> handles exception gracefully\n# @TEST_EDGE: get_remote_id_not_found -> returns None\n# @TEST_EDGE: get_batch_empty_list -> returns empty dict\n# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]\nclass IdMappingService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapping service.\n def __init__(self, db_session: Session):\n self.db = db_session\n self.scheduler = BackgroundScheduler()\n self._sync_job = None\n\n # [/DEF:__init__:Function]\n\n # [DEF:start_scheduler:Function]\n # @PURPOSE: Starts the background scheduler with a given cron string.\n # @PARAM: cron_string (str) - Cron expression for the sync interval.\n # @PARAM: environments (List[str]) - List of environment IDs to sync.\n # @PARAM: superset_client_factory - Function to get a client for an environment.\n def start_scheduler(\n self, cron_string: str, environments: List[str], superset_client_factory\n ):\n with belief_scope(\"IdMappingService.start_scheduler\"):\n if self._sync_job:\n self.scheduler.remove_job(self._sync_job.id)\n log.reflect(\"Removed existing sync job.\")\n\n def sync_all():\n for env_id in environments:\n client = superset_client_factory(env_id)\n if client:\n self.sync_environment(env_id, client)\n\n self._sync_job = self.scheduler.add_job(\n sync_all,\n CronTrigger.from_crontab(cron_string),\n id=\"id_mapping_sync_job\",\n replace_existing=True,\n )\n\n if not self.scheduler.running:\n self.scheduler.start()\n log.reason(f\"Started background scheduler with cron: {cron_string}\")\n else:\n log.reason(f\"Updated background scheduler with cron: {cron_string}\")\n\n # [/DEF:start_scheduler:Function]\n\n # [DEF:sync_environment:Function]\n # @PURPOSE: Fully synchronizes mapping for a specific environment.\n # @PARAM: environment_id (str) - Target environment ID.\n # @PARAM: superset_client - Instance capable of hitting the Superset API.\n # @PRE: environment_id exists in the database.\n # @POST: ResourceMapping records for the environment are created or updated.\n def sync_environment(\n self, environment_id: str, superset_client, incremental: bool = False\n ) -> None:\n \"\"\"\n Polls the Superset APIs for the target environment and updates the local mapping table.\n If incremental=True, only fetches items changed since the max last_synced_at date.\n \"\"\"\n with belief_scope(\"IdMappingService.sync_environment\"):\n log.reason(f\"Starting sync for environment {environment_id} (incremental={incremental})\")\n\n # Implementation Note: In a real scenario, superset_client needs to be an instance\n # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/\n # Here we structure the logic according to the spec.\n\n types_to_poll = [\n (ResourceType.CHART, \"chart\", \"slice_name\"),\n (ResourceType.DATASET, \"dataset\", \"table_name\"),\n (\n ResourceType.DASHBOARD,\n \"dashboard\",\n \"slug\",\n ), # Note: dashboard slug or dashboard_title\n ]\n\n total_synced = 0\n total_deleted = 0\n try:\n for res_enum, endpoint, name_field in types_to_poll:\n log.reason(f\"Polling {endpoint} endpoint\")\n\n # Simulated API Fetch (Would be: superset_client.get(f\"/api/v1/{endpoint}/\")... )\n # This relies on the superset API structure, e.g. { \"result\": [{\"id\": 1, \"uuid\": \"...\", name_field: \"...\"}] }\n # We assume superset_client provides a generic method to fetch all pages.\n\n try:\n since_dttm = None\n if incremental:\n from sqlalchemy.sql import func\n\n max_date = (\n self.db.query(func.max(ResourceMapping.last_synced_at))\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n .scalar()\n )\n\n if max_date:\n # We subtract a bit for safety overlap\n from datetime import timedelta\n\n since_dttm = max_date - timedelta(minutes=5)\n log.reason(\n f\"Incremental sync since {since_dttm}\"\n )\n\n resources = superset_client.get_all_resources(\n endpoint, since_dttm=since_dttm\n )\n\n # Track which UUIDs we see in this sync cycle\n synced_uuids = set()\n\n for res in resources:\n res_uuid = res.get(\"uuid\")\n raw_id = res.get(\"id\")\n res_name = res.get(name_field)\n\n if not res_uuid or raw_id is None:\n continue\n\n synced_uuids.add(res_uuid)\n res_id = str(raw_id) # Store as string\n\n # Upsert Logic\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n )\n .first()\n )\n\n if mapping:\n mapping.remote_integer_id = res_id\n mapping.resource_name = res_name\n mapping.last_synced_at = datetime.now(timezone.utc)\n else:\n new_mapping = ResourceMapping(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n remote_integer_id=res_id,\n resource_name=res_name,\n last_synced_at=datetime.now(timezone.utc),\n )\n self.db.add(new_mapping)\n\n total_synced += 1\n\n # Delete stale mappings: rows for this env+type whose UUID\n # was NOT returned by the API (resource was deleted remotely)\n # We only do this on full syncs, because incremental syncs don't return all UUIDs\n if not incremental:\n stale_query = self.db.query(ResourceMapping).filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n if synced_uuids:\n stale_query = stale_query.filter(\n ResourceMapping.uuid.notin_(synced_uuids)\n )\n deleted = stale_query.delete(synchronize_session=\"fetch\")\n if deleted:\n total_deleted += deleted\n log.reason(f\"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}\")\n\n except Exception as loop_e:\n log.explore(f\"Error polling {endpoint}\", error=str(loop_e))\n # Continue to next resource type instead of blowing up the whole sync\n\n self.db.commit()\n log.reflect(f\"Successfully synced {total_synced} items and deleted {total_deleted} stale items.\")\n\n except Exception as e:\n self.db.rollback()\n log.explore(\"Critical sync failure\", error=str(e))\n raise\n\n # [/DEF:sync_environment:Function]\n\n # [DEF:get_remote_id:Function]\n # @PURPOSE: Retrieves the remote integer ID for a given universal UUID.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuid (str)\n # @RETURN: Optional[int]\n def get_remote_id(\n self, environment_id: str, resource_type: ResourceType, uuid: str\n ) -> Optional[int]:\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id, resource_type=resource_type, uuid=uuid\n )\n .first()\n )\n\n if mapping:\n try:\n return int(mapping.remote_integer_id)\n except ValueError:\n return None\n return None\n\n # [/DEF:get_remote_id:Function]\n\n # [DEF:get_remote_ids_batch:Function]\n # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuids (List[str])\n # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID\n def get_remote_ids_batch(\n self, environment_id: str, resource_type: ResourceType, uuids: List[str]\n ) -> Dict[str, int]:\n if not uuids:\n return {}\n\n mappings = (\n self.db.query(ResourceMapping)\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == resource_type,\n ResourceMapping.uuid.in_(uuids),\n )\n .all()\n )\n\n result = {}\n for m in mappings:\n try:\n result[m.uuid] = int(m.remote_integer_id)\n except ValueError:\n pass\n\n return result\n\n # [/DEF:get_remote_ids_batch:Function]\n\n\n# [/DEF:IdMappingService:Class]\n" }, { "contract_id": "start_scheduler", "contract_type": "Function", "file_path": "backend/src/core/mapping_service.py", - "start_line": 63, - "end_line": 101, + "start_line": 66, + "end_line": 98, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -35783,14 +36252,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:start_scheduler:Function]\n # @PURPOSE: Starts the background scheduler with a given cron string.\n # @PARAM: cron_string (str) - Cron expression for the sync interval.\n # @PARAM: environments (List[str]) - List of environment IDs to sync.\n # @PARAM: superset_client_factory - Function to get a client for an environment.\n def start_scheduler(\n self, cron_string: str, environments: List[str], superset_client_factory\n ):\n with belief_scope(\"IdMappingService.start_scheduler\"):\n if self._sync_job:\n self.scheduler.remove_job(self._sync_job.id)\n logger.info(\n \"[IdMappingService.start_scheduler][Reflect] Removed existing sync job.\"\n )\n\n def sync_all():\n for env_id in environments:\n client = superset_client_factory(env_id)\n if client:\n self.sync_environment(env_id, client)\n\n self._sync_job = self.scheduler.add_job(\n sync_all,\n CronTrigger.from_crontab(cron_string),\n id=\"id_mapping_sync_job\",\n replace_existing=True,\n )\n\n if not self.scheduler.running:\n self.scheduler.start()\n logger.info(\n f\"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}\"\n )\n else:\n logger.info(\n f\"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}\"\n )\n\n # [/DEF:start_scheduler:Function]\n" + "body": " # [DEF:start_scheduler:Function]\n # @PURPOSE: Starts the background scheduler with a given cron string.\n # @PARAM: cron_string (str) - Cron expression for the sync interval.\n # @PARAM: environments (List[str]) - List of environment IDs to sync.\n # @PARAM: superset_client_factory - Function to get a client for an environment.\n def start_scheduler(\n self, cron_string: str, environments: List[str], superset_client_factory\n ):\n with belief_scope(\"IdMappingService.start_scheduler\"):\n if self._sync_job:\n self.scheduler.remove_job(self._sync_job.id)\n log.reflect(\"Removed existing sync job.\")\n\n def sync_all():\n for env_id in environments:\n client = superset_client_factory(env_id)\n if client:\n self.sync_environment(env_id, client)\n\n self._sync_job = self.scheduler.add_job(\n sync_all,\n CronTrigger.from_crontab(cron_string),\n id=\"id_mapping_sync_job\",\n replace_existing=True,\n )\n\n if not self.scheduler.running:\n self.scheduler.start()\n log.reason(f\"Started background scheduler with cron: {cron_string}\")\n else:\n log.reason(f\"Updated background scheduler with cron: {cron_string}\")\n\n # [/DEF:start_scheduler:Function]\n" }, { "contract_id": "sync_environment", "contract_type": "Function", "file_path": "backend/src/core/mapping_service.py", - "start_line": 103, - "end_line": 253, + "start_line": 100, + "end_line": 238, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -35827,14 +36296,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:sync_environment:Function]\n # @PURPOSE: Fully synchronizes mapping for a specific environment.\n # @PARAM: environment_id (str) - Target environment ID.\n # @PARAM: superset_client - Instance capable of hitting the Superset API.\n # @PRE: environment_id exists in the database.\n # @POST: ResourceMapping records for the environment are created or updated.\n def sync_environment(\n self, environment_id: str, superset_client, incremental: bool = False\n ) -> None:\n \"\"\"\n Polls the Superset APIs for the target environment and updates the local mapping table.\n If incremental=True, only fetches items changed since the max last_synced_at date.\n \"\"\"\n with belief_scope(\"IdMappingService.sync_environment\"):\n logger.info(\n f\"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})\"\n )\n\n # Implementation Note: In a real scenario, superset_client needs to be an instance\n # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/\n # Here we structure the logic according to the spec.\n\n types_to_poll = [\n (ResourceType.CHART, \"chart\", \"slice_name\"),\n (ResourceType.DATASET, \"dataset\", \"table_name\"),\n (\n ResourceType.DASHBOARD,\n \"dashboard\",\n \"slug\",\n ), # Note: dashboard slug or dashboard_title\n ]\n\n total_synced = 0\n total_deleted = 0\n try:\n for res_enum, endpoint, name_field in types_to_poll:\n logger.debug(\n f\"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint\"\n )\n\n # Simulated API Fetch (Would be: superset_client.get(f\"/api/v1/{endpoint}/\")... )\n # This relies on the superset API structure, e.g. { \"result\": [{\"id\": 1, \"uuid\": \"...\", name_field: \"...\"}] }\n # We assume superset_client provides a generic method to fetch all pages.\n\n try:\n since_dttm = None\n if incremental:\n from sqlalchemy.sql import func\n\n max_date = (\n self.db.query(func.max(ResourceMapping.last_synced_at))\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n .scalar()\n )\n\n if max_date:\n # We subtract a bit for safety overlap\n from datetime import timedelta\n\n since_dttm = max_date - timedelta(minutes=5)\n logger.debug(\n f\"[IdMappingService.sync_environment] Incremental sync since {since_dttm}\"\n )\n\n resources = superset_client.get_all_resources(\n endpoint, since_dttm=since_dttm\n )\n\n # Track which UUIDs we see in this sync cycle\n synced_uuids = set()\n\n for res in resources:\n res_uuid = res.get(\"uuid\")\n raw_id = res.get(\"id\")\n res_name = res.get(name_field)\n\n if not res_uuid or raw_id is None:\n continue\n\n synced_uuids.add(res_uuid)\n res_id = str(raw_id) # Store as string\n\n # Upsert Logic\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n )\n .first()\n )\n\n if mapping:\n mapping.remote_integer_id = res_id\n mapping.resource_name = res_name\n mapping.last_synced_at = datetime.now(timezone.utc)\n else:\n new_mapping = ResourceMapping(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n remote_integer_id=res_id,\n resource_name=res_name,\n last_synced_at=datetime.now(timezone.utc),\n )\n self.db.add(new_mapping)\n\n total_synced += 1\n\n # Delete stale mappings: rows for this env+type whose UUID\n # was NOT returned by the API (resource was deleted remotely)\n # We only do this on full syncs, because incremental syncs don't return all UUIDs\n if not incremental:\n stale_query = self.db.query(ResourceMapping).filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n if synced_uuids:\n stale_query = stale_query.filter(\n ResourceMapping.uuid.notin_(synced_uuids)\n )\n deleted = stale_query.delete(synchronize_session=\"fetch\")\n if deleted:\n total_deleted += deleted\n logger.info(\n f\"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}\"\n )\n\n except Exception as loop_e:\n logger.error(\n f\"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}\"\n )\n # Continue to next resource type instead of blowing up the whole sync\n\n self.db.commit()\n logger.info(\n f\"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items.\"\n )\n\n except Exception as e:\n self.db.rollback()\n logger.error(\n f\"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}\"\n )\n raise\n\n # [/DEF:sync_environment:Function]\n" + "body": " # [DEF:sync_environment:Function]\n # @PURPOSE: Fully synchronizes mapping for a specific environment.\n # @PARAM: environment_id (str) - Target environment ID.\n # @PARAM: superset_client - Instance capable of hitting the Superset API.\n # @PRE: environment_id exists in the database.\n # @POST: ResourceMapping records for the environment are created or updated.\n def sync_environment(\n self, environment_id: str, superset_client, incremental: bool = False\n ) -> None:\n \"\"\"\n Polls the Superset APIs for the target environment and updates the local mapping table.\n If incremental=True, only fetches items changed since the max last_synced_at date.\n \"\"\"\n with belief_scope(\"IdMappingService.sync_environment\"):\n log.reason(f\"Starting sync for environment {environment_id} (incremental={incremental})\")\n\n # Implementation Note: In a real scenario, superset_client needs to be an instance\n # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/\n # Here we structure the logic according to the spec.\n\n types_to_poll = [\n (ResourceType.CHART, \"chart\", \"slice_name\"),\n (ResourceType.DATASET, \"dataset\", \"table_name\"),\n (\n ResourceType.DASHBOARD,\n \"dashboard\",\n \"slug\",\n ), # Note: dashboard slug or dashboard_title\n ]\n\n total_synced = 0\n total_deleted = 0\n try:\n for res_enum, endpoint, name_field in types_to_poll:\n log.reason(f\"Polling {endpoint} endpoint\")\n\n # Simulated API Fetch (Would be: superset_client.get(f\"/api/v1/{endpoint}/\")... )\n # This relies on the superset API structure, e.g. { \"result\": [{\"id\": 1, \"uuid\": \"...\", name_field: \"...\"}] }\n # We assume superset_client provides a generic method to fetch all pages.\n\n try:\n since_dttm = None\n if incremental:\n from sqlalchemy.sql import func\n\n max_date = (\n self.db.query(func.max(ResourceMapping.last_synced_at))\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n .scalar()\n )\n\n if max_date:\n # We subtract a bit for safety overlap\n from datetime import timedelta\n\n since_dttm = max_date - timedelta(minutes=5)\n log.reason(\n f\"Incremental sync since {since_dttm}\"\n )\n\n resources = superset_client.get_all_resources(\n endpoint, since_dttm=since_dttm\n )\n\n # Track which UUIDs we see in this sync cycle\n synced_uuids = set()\n\n for res in resources:\n res_uuid = res.get(\"uuid\")\n raw_id = res.get(\"id\")\n res_name = res.get(name_field)\n\n if not res_uuid or raw_id is None:\n continue\n\n synced_uuids.add(res_uuid)\n res_id = str(raw_id) # Store as string\n\n # Upsert Logic\n mapping = (\n self.db.query(ResourceMapping)\n .filter_by(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n )\n .first()\n )\n\n if mapping:\n mapping.remote_integer_id = res_id\n mapping.resource_name = res_name\n mapping.last_synced_at = datetime.now(timezone.utc)\n else:\n new_mapping = ResourceMapping(\n environment_id=environment_id,\n resource_type=res_enum,\n uuid=res_uuid,\n remote_integer_id=res_id,\n resource_name=res_name,\n last_synced_at=datetime.now(timezone.utc),\n )\n self.db.add(new_mapping)\n\n total_synced += 1\n\n # Delete stale mappings: rows for this env+type whose UUID\n # was NOT returned by the API (resource was deleted remotely)\n # We only do this on full syncs, because incremental syncs don't return all UUIDs\n if not incremental:\n stale_query = self.db.query(ResourceMapping).filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == res_enum,\n )\n if synced_uuids:\n stale_query = stale_query.filter(\n ResourceMapping.uuid.notin_(synced_uuids)\n )\n deleted = stale_query.delete(synchronize_session=\"fetch\")\n if deleted:\n total_deleted += deleted\n log.reason(f\"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}\")\n\n except Exception as loop_e:\n log.explore(f\"Error polling {endpoint}\", error=str(loop_e))\n # Continue to next resource type instead of blowing up the whole sync\n\n self.db.commit()\n log.reflect(f\"Successfully synced {total_synced} items and deleted {total_deleted} stale items.\")\n\n except Exception as e:\n self.db.rollback()\n log.explore(\"Critical sync failure\", error=str(e))\n raise\n\n # [/DEF:sync_environment:Function]\n" }, { "contract_id": "get_remote_id", "contract_type": "Function", "file_path": "backend/src/core/mapping_service.py", - "start_line": 255, - "end_line": 279, + "start_line": 240, + "end_line": 264, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -35864,8 +36333,8 @@ "contract_id": "get_remote_ids_batch", "contract_type": "Function", "file_path": "backend/src/core/mapping_service.py", - "start_line": 281, - "end_line": 312, + "start_line": 266, + "end_line": 297, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -35891,6 +36360,293 @@ "anchor_syntax": "def", "body": " # [DEF:get_remote_ids_batch:Function]\n # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.\n # @PARAM: environment_id (str)\n # @PARAM: resource_type (ResourceType)\n # @PARAM: uuids (List[str])\n # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID\n def get_remote_ids_batch(\n self, environment_id: str, resource_type: ResourceType, uuids: List[str]\n ) -> Dict[str, int]:\n if not uuids:\n return {}\n\n mappings = (\n self.db.query(ResourceMapping)\n .filter(\n ResourceMapping.environment_id == environment_id,\n ResourceMapping.resource_type == resource_type,\n ResourceMapping.uuid.in_(uuids),\n )\n .all()\n )\n\n result = {}\n for m in mappings:\n try:\n result[m.uuid] = int(m.remote_integer_id)\n except ValueError:\n pass\n\n return result\n\n # [/DEF:get_remote_ids_batch:Function]\n" }, + { + "contract_id": "middleware_package", + "contract_type": "Package", + "file_path": "backend/src/core/middleware/__init__.py", + "start_line": 1, + "end_line": 5, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1, + "PURPOSE": "FastAPI/Starlette middleware package for request-level context and tracing.", + "SEMANTICS": [ + "middleware", + "package", + "starlette", + "fastapi" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "tag_not_for_contract_type", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is not allowed for contract type 'Package'", + "detail": { + "actual_type": "Package", + "allowed_types": [ + "Module", + "Function", + "Class", + "Component", + "Block" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "COMPLEXITY", + "message": "@COMPLEXITY is forbidden for contract type 'Package' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Package" + } + }, + { + "code": "tag_not_for_contract_type", + "tag": "PURPOSE", + "message": "@PURPOSE is not allowed for contract type 'Package'", + "detail": { + "actual_type": "Package", + "allowed_types": [ + "Module", + "Function", + "Class", + "Component", + "Block", + "ADR" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "PURPOSE", + "message": "@PURPOSE is forbidden for contract type 'Package' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Package" + } + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Package'", + "detail": { + "actual_type": "Package", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Package' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Package" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:middleware_package:Package]\n# @COMPLEXITY: 1\n# @SEMANTICS: middleware, package, starlette, fastapi\n# @PURPOSE: FastAPI/Starlette middleware package for request-level context and tracing.\n# [/DEF:middleware_package:Package]\n" + }, + { + "contract_id": "TraceContextMiddlewareModule", + "contract_type": "Module", + "file_path": "backend/src/core/middleware/trace.py", + "start_line": 1, + "end_line": 77, + "tier": "TIER_2", + "complexity": 3, + "metadata": { + "COMPLEXITY": 3, + "PURPOSE": "FastAPI/Starlette middleware that seeds a trace_id for every incoming HTTP request.", + "SEMANTICS": [ + "middleware", + "trace", + "context", + "starlette", + "fastapi" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "missing_required_tag", + "tag": "LAYER", + "message": "@LAYER is required for contract type 'Module' at C3", + "detail": { + "actual_complexity": 3, + "contract_type": "Module" + } + }, + { + "code": "missing_required_tag", + "tag": "RELATION", + "message": "@RELATION is required for contract type 'Module' at C3", + "detail": { + "actual_complexity": 3, + "contract_type": "Module" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:TraceContextMiddlewareModule:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: middleware, trace, context, starlette, fastapi\n# @PURPOSE: FastAPI/Starlette middleware that seeds a trace_id for every incoming HTTP request.\n# Optionally extracts X-Trace-ID header for cross-service trace propagation.\n# @LAYER: Core\n# @RELATION: [DEPENDS_ON] -> [CotLoggerModule]\n# @RELATION: [CALLED_BY] -> [AppModule]\n# @PRE: FastAPI app instance with Starlette middleware support.\n# @POST: Every request gets a trace_id via seed_trace_id(). Existing X-Trace-ID header is\n# preserved and used as the trace_id when present.\n# @SIDE_EFFECT: Sets ContextVar _trace_id for the duration of the request.\n\nfrom starlette.middleware.base import BaseHTTPMiddleware\nfrom starlette.requests import Request\nfrom starlette.responses import Response\nfrom starlette.types import ASGIApp\n\nfrom ..cot_logger import seed_trace_id, set_trace_id\n\n# [DEF:TraceContextMiddleware:Class]\n# @COMPLEXITY: 2\n# @SEMANTICS: middleware, trace, context, dispatch\n# @BRIEF: Starlette BaseHTTPMiddleware that seeds a trace_id per request.\nclass TraceContextMiddleware(BaseHTTPMiddleware):\n \"\"\"FastAPI/Starlette middleware that seeds a trace_id for every request.\n\n If the incoming request carries an ``X-Trace-ID`` header, that value is\n used as the trace_id (enabling cross-service trace chaining). Otherwise a\n new UUID4 is generated.\n\n Usage::\n\n from backend.src.core.middleware.trace import TraceContextMiddleware\n app.add_middleware(TraceContextMiddleware)\n \"\"\"\n\n # [DEF:TraceContextMiddleware.__init__:Function]\n # @BRIEF: Standard BaseHTTPMiddleware initialiser.\n def __init__(self, app: ASGIApp) -> None:\n super().__init__(app)\n\n # [/DEF:TraceContextMiddleware.__init__:Function]\n\n # [DEF:TraceContextMiddleware.dispatch:Function]\n # @COMPLEXITY: 2\n # @SEMANTICS: dispatch, trace, seed, header, call_next\n # @BRIEF: Dispatch handler that seeds trace_id before passing to the next middleware.\n async def dispatch(\n self, request: Request, call_next\n ) -> Response:\n \"\"\"Intercept every request, seed the trace_id, and forward.\n\n If the client sent an ``X-Trace-ID`` header, use it; otherwise\n ``seed_trace_id()`` generates a new UUID4.\n\n Args:\n request: The incoming Starlette Request.\n call_next: The next middleware or route handler.\n\n Returns:\n Response from the downstream handler.\n \"\"\"\n incoming_trace_id = request.headers.get(\"X-Trace-ID\")\n if incoming_trace_id:\n set_trace_id(incoming_trace_id)\n else:\n seed_trace_id()\n\n response = await call_next(request)\n return response\n\n # [/DEF:TraceContextMiddleware.dispatch:Function]\n\n\n# [/DEF:TraceContextMiddleware:Class]\n# [/DEF:TraceContextMiddlewareModule:Module]\n" + }, + { + "contract_id": "TraceContextMiddleware", + "contract_type": "Class", + "file_path": "backend/src/core/middleware/trace.py", + "start_line": 21, + "end_line": 76, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Starlette BaseHTTPMiddleware that seeds a trace_id per request.", + "COMPLEXITY": 2, + "SEMANTICS": [ + "middleware", + "trace", + "context", + "dispatch" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Class'", + "detail": { + "actual_type": "Class", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Class' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Class" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Class' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Class" + } + } + ], + "anchor_syntax": "def", + "body": "# [DEF:TraceContextMiddleware:Class]\n# @COMPLEXITY: 2\n# @SEMANTICS: middleware, trace, context, dispatch\n# @BRIEF: Starlette BaseHTTPMiddleware that seeds a trace_id per request.\nclass TraceContextMiddleware(BaseHTTPMiddleware):\n \"\"\"FastAPI/Starlette middleware that seeds a trace_id for every request.\n\n If the incoming request carries an ``X-Trace-ID`` header, that value is\n used as the trace_id (enabling cross-service trace chaining). Otherwise a\n new UUID4 is generated.\n\n Usage::\n\n from backend.src.core.middleware.trace import TraceContextMiddleware\n app.add_middleware(TraceContextMiddleware)\n \"\"\"\n\n # [DEF:TraceContextMiddleware.__init__:Function]\n # @BRIEF: Standard BaseHTTPMiddleware initialiser.\n def __init__(self, app: ASGIApp) -> None:\n super().__init__(app)\n\n # [/DEF:TraceContextMiddleware.__init__:Function]\n\n # [DEF:TraceContextMiddleware.dispatch:Function]\n # @COMPLEXITY: 2\n # @SEMANTICS: dispatch, trace, seed, header, call_next\n # @BRIEF: Dispatch handler that seeds trace_id before passing to the next middleware.\n async def dispatch(\n self, request: Request, call_next\n ) -> Response:\n \"\"\"Intercept every request, seed the trace_id, and forward.\n\n If the client sent an ``X-Trace-ID`` header, use it; otherwise\n ``seed_trace_id()`` generates a new UUID4.\n\n Args:\n request: The incoming Starlette Request.\n call_next: The next middleware or route handler.\n\n Returns:\n Response from the downstream handler.\n \"\"\"\n incoming_trace_id = request.headers.get(\"X-Trace-ID\")\n if incoming_trace_id:\n set_trace_id(incoming_trace_id)\n else:\n seed_trace_id()\n\n response = await call_next(request)\n return response\n\n # [/DEF:TraceContextMiddleware.dispatch:Function]\n\n\n# [/DEF:TraceContextMiddleware:Class]\n" + }, + { + "contract_id": "TraceContextMiddleware.__init__", + "contract_type": "Function", + "file_path": "backend/src/core/middleware/trace.py", + "start_line": 38, + "end_line": 43, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "BRIEF": "Standard BaseHTTPMiddleware initialiser." + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": " # [DEF:TraceContextMiddleware.__init__:Function]\n # @BRIEF: Standard BaseHTTPMiddleware initialiser.\n def __init__(self, app: ASGIApp) -> None:\n super().__init__(app)\n\n # [/DEF:TraceContextMiddleware.__init__:Function]\n" + }, + { + "contract_id": "TraceContextMiddleware.dispatch", + "contract_type": "Function", + "file_path": "backend/src/core/middleware/trace.py", + "start_line": 45, + "end_line": 73, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Dispatch handler that seeds trace_id before passing to the next middleware.", + "COMPLEXITY": 2, + "SEMANTICS": [ + "dispatch", + "trace", + "seed", + "header", + "call_next" + ] + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "tag_not_for_contract_type", + "tag": "SEMANTICS", + "message": "@SEMANTICS is not allowed for contract type 'Function'", + "detail": { + "actual_type": "Function", + "allowed_types": [ + "Module" + ] + } + }, + { + "code": "tag_forbidden_by_complexity", + "tag": "SEMANTICS", + "message": "@SEMANTICS is forbidden for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "def", + "body": " # [DEF:TraceContextMiddleware.dispatch:Function]\n # @COMPLEXITY: 2\n # @SEMANTICS: dispatch, trace, seed, header, call_next\n # @BRIEF: Dispatch handler that seeds trace_id before passing to the next middleware.\n async def dispatch(\n self, request: Request, call_next\n ) -> Response:\n \"\"\"Intercept every request, seed the trace_id, and forward.\n\n If the client sent an ``X-Trace-ID`` header, use it; otherwise\n ``seed_trace_id()`` generates a new UUID4.\n\n Args:\n request: The incoming Starlette Request.\n call_next: The next middleware or route handler.\n\n Returns:\n Response from the downstream handler.\n \"\"\"\n incoming_trace_id = request.headers.get(\"X-Trace-ID\")\n if incoming_trace_id:\n set_trace_id(incoming_trace_id)\n else:\n seed_trace_id()\n\n response = await call_next(request)\n return response\n\n # [/DEF:TraceContextMiddleware.dispatch:Function]\n" + }, { "contract_id": "MigrationPackage", "contract_type": "Module", @@ -36631,7 +37387,7 @@ "contract_type": "Module", "file_path": "backend/src/core/migration/risk_assessor.py", "start_line": 1, - "end_line": 202, + "end_line": 205, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -36840,14 +37596,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:RiskAssessorModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: migration, dry_run, risk, scoring, preflight\n# @PURPOSE: Compute deterministic migration risk items and aggregate score for dry-run reporting.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [SupersetClient]\n# @RELATION: CONTAINS -> [index_by_uuid]\n# @RELATION: CONTAINS -> [extract_owner_identifiers]\n# @RELATION: CONTAINS -> [build_risks]\n# @RELATION: CONTAINS -> [score_risks]\n# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.\n# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration.\n# @POST: Risk scoring output preserves item list and provides bounded score with derived level.\n# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client.\n# @DATA_CONTRACT: Module[build_risks, score_risks]\n# @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]\n# @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item]\n# @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted]\n# @TEST_SCENARIO: [owner_mismatch_dashboard] -> [low owner_mismatch risk is emitted]\n# @TEST_EDGE: [missing_field] -> [object without uuid is ignored by indexer]\n# @TEST_EDGE: [invalid_type] -> [non-list owners input normalizes to empty identifiers]\n# @TEST_EDGE: [external_fail] -> [target_client get_databases exception propagates to caller]\n# @TEST_INVARIANT: [score_upper_bound_100] -> VERIFIED_BY: [severity_weight_aggregation]\n# @UX_STATE: [Idle] -> [N/A backend domain module]\n# @UX_FEEDBACK: [N/A] -> [No direct UI side effects in this module]\n# @UX_RECOVERY: [N/A] -> [Caller-level retry/recovery]\n# @UX_REACTIVITY: [N/A] -> [Backend synchronous function contracts]\n\nfrom typing import Any, Dict, List\n\nfrom ..logger import logger, belief_scope\nfrom ..superset_client import SupersetClient\n\n\n# [DEF:index_by_uuid:Function]\n# @PURPOSE: Build UUID-index from normalized objects.\n# @PRE: Input list items are dict-like payloads potentially containing \"uuid\".\n# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]\ndef index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:\n with belief_scope(\"risk_assessor.index_by_uuid\"):\n logger.reason(\"Building UUID index\", extra={\"objects_count\": len(objects)})\n indexed: Dict[str, Dict[str, Any]] = {}\n for obj in objects:\n uuid = obj.get(\"uuid\")\n if uuid:\n indexed[str(uuid)] = obj\n logger.reflect(\"UUID index built\", extra={\"indexed_count\": len(indexed)})\n return indexed\n\n\n# [/DEF:index_by_uuid:Function]\n\n\n# [DEF:extract_owner_identifiers:Function]\n# @PURPOSE: Normalize owner payloads for stable comparison.\n# @PRE: Owners may be list payload, scalar values, or None.\n# @POST: Returns sorted unique owner identifiers as strings.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: Any -> List[str]\ndef extract_owner_identifiers(owners: Any) -> List[str]:\n with belief_scope(\"risk_assessor.extract_owner_identifiers\"):\n logger.reason(\"Normalizing owner identifiers\")\n if not isinstance(owners, list):\n logger.reflect(\"Owners payload is not list; returning empty identifiers\")\n return []\n ids: List[str] = []\n for owner in owners:\n if isinstance(owner, dict):\n if owner.get(\"username\"):\n ids.append(str(owner[\"username\"]))\n elif owner.get(\"id\") is not None:\n ids.append(str(owner[\"id\"]))\n elif owner is not None:\n ids.append(str(owner))\n normalized_ids = sorted(set(ids))\n logger.reflect(\n \"Owner identifiers normalized\", extra={\"owner_count\": len(normalized_ids)}\n )\n return normalized_ids\n\n\n# [/DEF:extract_owner_identifiers:Function]\n\n\n# [DEF:build_risks:Function]\n# @PURPOSE: Build risk list from computed diffs and target catalog state.\n# @RELATION: DEPENDS_ON -> [index_by_uuid]\n# @RELATION: DEPENDS_ON -> [extract_owner_identifiers]\n# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.\n# @PRE: target_client is authenticated/usable for database list retrieval.\n# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.\n# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs.\n# @DATA_CONTRACT: (\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, Dict[str, List[Dict[str, Any]]]],\n# @DATA_CONTRACT: SupersetClient\n# @DATA_CONTRACT: ) -> List[Dict[str, Any]]\ndef build_risks(\n source_objects: Dict[str, List[Dict[str, Any]]],\n target_objects: Dict[str, List[Dict[str, Any]]],\n diff: Dict[str, Dict[str, List[Dict[str, Any]]]],\n target_client: SupersetClient,\n) -> List[Dict[str, Any]]:\n with belief_scope(\"risk_assessor.build_risks\"):\n logger.reason(\"Building migration risks from diff payload\")\n risks: List[Dict[str, Any]] = []\n for object_type in (\"dashboards\", \"charts\", \"datasets\"):\n for item in diff[object_type][\"update\"]:\n risks.append(\n {\n \"code\": \"overwrite_existing\",\n \"severity\": \"medium\",\n \"object_type\": object_type[:-1],\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Object will be updated in target: {item.get('title') or item['uuid']}\",\n }\n )\n\n target_dataset_uuids = set(index_by_uuid(target_objects[\"datasets\"]).keys())\n _, target_databases = target_client.get_databases(query={\"columns\": [\"uuid\"]})\n target_database_uuids = {\n str(item.get(\"uuid\")) for item in target_databases if item.get(\"uuid\")\n }\n\n for dataset in source_objects[\"datasets\"]:\n db_uuid = dataset.get(\"database_uuid\")\n if db_uuid and str(db_uuid) not in target_database_uuids:\n risks.append(\n {\n \"code\": \"missing_datasource\",\n \"severity\": \"high\",\n \"object_type\": \"dataset\",\n \"object_uuid\": dataset.get(\"uuid\"),\n \"message\": f\"Target datasource is missing for dataset {dataset.get('title') or dataset.get('uuid')}\",\n }\n )\n\n for chart in source_objects[\"charts\"]:\n ds_uuid = chart.get(\"dataset_uuid\")\n if ds_uuid and str(ds_uuid) not in target_dataset_uuids:\n risks.append(\n {\n \"code\": \"breaking_reference\",\n \"severity\": \"high\",\n \"object_type\": \"chart\",\n \"object_uuid\": chart.get(\"uuid\"),\n \"message\": f\"Chart references dataset not found on target: {ds_uuid}\",\n }\n )\n\n source_dash = index_by_uuid(source_objects[\"dashboards\"])\n target_dash = index_by_uuid(target_objects[\"dashboards\"])\n for item in diff[\"dashboards\"][\"update\"]:\n source_obj = source_dash.get(item[\"uuid\"])\n target_obj = target_dash.get(item[\"uuid\"])\n if not source_obj or not target_obj:\n continue\n source_owners = extract_owner_identifiers(source_obj.get(\"owners\"))\n target_owners = extract_owner_identifiers(target_obj.get(\"owners\"))\n if source_owners and target_owners and source_owners != target_owners:\n risks.append(\n {\n \"code\": \"owner_mismatch\",\n \"severity\": \"low\",\n \"object_type\": \"dashboard\",\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Owner mismatch for dashboard {item.get('title') or item['uuid']}\",\n }\n )\n logger.reflect(\"Risk list assembled\", extra={\"risk_count\": len(risks)})\n return risks\n\n\n# [/DEF:build_risks:Function]\n\n\n# [DEF:score_risks:Function]\n# @PURPOSE: Aggregate risk list into score and level.\n# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.\n# @POST: Returns dict with score in [0,100], derived level, and original items.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any]\ndef score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope(\"risk_assessor.score_risks\"):\n logger.reason(\"Scoring risk items\", extra={\"risk_items_count\": len(risk_items)})\n weights = {\"high\": 25, \"medium\": 10, \"low\": 5}\n score = min(\n 100, sum(weights.get(item.get(\"severity\", \"low\"), 5) for item in risk_items)\n )\n level = \"low\" if score < 25 else \"medium\" if score < 60 else \"high\"\n result = {\"score\": score, \"level\": level, \"items\": risk_items}\n logger.reflect(\"Risk score computed\", extra={\"score\": score, \"level\": level})\n return result\n\n\n# [/DEF:score_risks:Function]\n\n\n# [/DEF:RiskAssessorModule:Module]\n" + "body": "# [DEF:RiskAssessorModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: migration, dry_run, risk, scoring, preflight\n# @PURPOSE: Compute deterministic migration risk items and aggregate score for dry-run reporting.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [SupersetClient]\n# @RELATION: CONTAINS -> [index_by_uuid]\n# @RELATION: CONTAINS -> [extract_owner_identifiers]\n# @RELATION: CONTAINS -> [build_risks]\n# @RELATION: CONTAINS -> [score_risks]\n# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.\n# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration.\n# @POST: Risk scoring output preserves item list and provides bounded score with derived level.\n# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client.\n# @DATA_CONTRACT: Module[build_risks, score_risks]\n# @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]\n# @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item]\n# @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted]\n# @TEST_SCENARIO: [owner_mismatch_dashboard] -> [low owner_mismatch risk is emitted]\n# @TEST_EDGE: [missing_field] -> [object without uuid is ignored by indexer]\n# @TEST_EDGE: [invalid_type] -> [non-list owners input normalizes to empty identifiers]\n# @TEST_EDGE: [external_fail] -> [target_client get_databases exception propagates to caller]\n# @TEST_INVARIANT: [score_upper_bound_100] -> VERIFIED_BY: [severity_weight_aggregation]\n# @UX_STATE: [Idle] -> [N/A backend domain module]\n# @UX_FEEDBACK: [N/A] -> [No direct UI side effects in this module]\n# @UX_RECOVERY: [N/A] -> [Caller-level retry/recovery]\n# @UX_REACTIVITY: [N/A] -> [Backend synchronous function contracts]\n\nfrom typing import Any, Dict, List\n\nfrom ..logger import belief_scope\nfrom ..cot_logger import MarkerLogger\nfrom ..superset_client import SupersetClient\n\nlog = MarkerLogger(\"RiskAssessor\")\n\n\n# [DEF:index_by_uuid:Function]\n# @PURPOSE: Build UUID-index from normalized objects.\n# @PRE: Input list items are dict-like payloads potentially containing \"uuid\".\n# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]\ndef index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:\n with belief_scope(\"risk_assessor.index_by_uuid\"):\n log.reason(\"Building UUID index\", payload={\"objects_count\": len(objects)})\n indexed: Dict[str, Dict[str, Any]] = {}\n for obj in objects:\n uuid = obj.get(\"uuid\")\n if uuid:\n indexed[str(uuid)] = obj\n log.reflect(\"UUID index built\", payload={\"indexed_count\": len(indexed)})\n return indexed\n\n\n# [/DEF:index_by_uuid:Function]\n\n\n# [DEF:extract_owner_identifiers:Function]\n# @PURPOSE: Normalize owner payloads for stable comparison.\n# @PRE: Owners may be list payload, scalar values, or None.\n# @POST: Returns sorted unique owner identifiers as strings.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: Any -> List[str]\ndef extract_owner_identifiers(owners: Any) -> List[str]:\n with belief_scope(\"risk_assessor.extract_owner_identifiers\"):\n log.reason(\"Normalizing owner identifiers\")\n if not isinstance(owners, list):\n log.reflect(\"Owners payload is not list; returning empty identifiers\")\n return []\n ids: List[str] = []\n for owner in owners:\n if isinstance(owner, dict):\n if owner.get(\"username\"):\n ids.append(str(owner[\"username\"]))\n elif owner.get(\"id\") is not None:\n ids.append(str(owner[\"id\"]))\n elif owner is not None:\n ids.append(str(owner))\n normalized_ids = sorted(set(ids))\n log.reflect(\n \"Owner identifiers normalized\", payload={\"owner_count\": len(normalized_ids)}\n )\n return normalized_ids\n\n\n# [/DEF:extract_owner_identifiers:Function]\n\n\n# [DEF:build_risks:Function]\n# @PURPOSE: Build risk list from computed diffs and target catalog state.\n# @RELATION: DEPENDS_ON -> [index_by_uuid]\n# @RELATION: DEPENDS_ON -> [extract_owner_identifiers]\n# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.\n# @PRE: target_client is authenticated/usable for database list retrieval.\n# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.\n# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs.\n# @DATA_CONTRACT: (\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, Dict[str, List[Dict[str, Any]]]],\n# @DATA_CONTRACT: SupersetClient\n# @DATA_CONTRACT: ) -> List[Dict[str, Any]]\ndef build_risks(\n source_objects: Dict[str, List[Dict[str, Any]]],\n target_objects: Dict[str, List[Dict[str, Any]]],\n diff: Dict[str, Dict[str, List[Dict[str, Any]]]],\n target_client: SupersetClient,\n) -> List[Dict[str, Any]]:\n with belief_scope(\"risk_assessor.build_risks\"):\n log.reason(\"Building migration risks from diff payload\")\n risks: List[Dict[str, Any]] = []\n for object_type in (\"dashboards\", \"charts\", \"datasets\"):\n for item in diff[object_type][\"update\"]:\n risks.append(\n {\n \"code\": \"overwrite_existing\",\n \"severity\": \"medium\",\n \"object_type\": object_type[:-1],\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Object will be updated in target: {item.get('title') or item['uuid']}\",\n }\n )\n\n target_dataset_uuids = set(index_by_uuid(target_objects[\"datasets\"]).keys())\n _, target_databases = target_client.get_databases(query={\"columns\": [\"uuid\"]})\n target_database_uuids = {\n str(item.get(\"uuid\")) for item in target_databases if item.get(\"uuid\")\n }\n\n for dataset in source_objects[\"datasets\"]:\n db_uuid = dataset.get(\"database_uuid\")\n if db_uuid and str(db_uuid) not in target_database_uuids:\n risks.append(\n {\n \"code\": \"missing_datasource\",\n \"severity\": \"high\",\n \"object_type\": \"dataset\",\n \"object_uuid\": dataset.get(\"uuid\"),\n \"message\": f\"Target datasource is missing for dataset {dataset.get('title') or dataset.get('uuid')}\",\n }\n )\n\n for chart in source_objects[\"charts\"]:\n ds_uuid = chart.get(\"dataset_uuid\")\n if ds_uuid and str(ds_uuid) not in target_dataset_uuids:\n risks.append(\n {\n \"code\": \"breaking_reference\",\n \"severity\": \"high\",\n \"object_type\": \"chart\",\n \"object_uuid\": chart.get(\"uuid\"),\n \"message\": f\"Chart references dataset not found on target: {ds_uuid}\",\n }\n )\n\n source_dash = index_by_uuid(source_objects[\"dashboards\"])\n target_dash = index_by_uuid(target_objects[\"dashboards\"])\n for item in diff[\"dashboards\"][\"update\"]:\n source_obj = source_dash.get(item[\"uuid\"])\n target_obj = target_dash.get(item[\"uuid\"])\n if not source_obj or not target_obj:\n continue\n source_owners = extract_owner_identifiers(source_obj.get(\"owners\"))\n target_owners = extract_owner_identifiers(target_obj.get(\"owners\"))\n if source_owners and target_owners and source_owners != target_owners:\n risks.append(\n {\n \"code\": \"owner_mismatch\",\n \"severity\": \"low\",\n \"object_type\": \"dashboard\",\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Owner mismatch for dashboard {item.get('title') or item['uuid']}\",\n }\n )\n log.reflect(\"Risk list assembled\", payload={\"risk_count\": len(risks)})\n return risks\n\n\n# [/DEF:build_risks:Function]\n\n\n# [DEF:score_risks:Function]\n# @PURPOSE: Aggregate risk list into score and level.\n# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.\n# @POST: Returns dict with score in [0,100], derived level, and original items.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any]\ndef score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope(\"risk_assessor.score_risks\"):\n log.reason(\"Scoring risk items\", payload={\"risk_items_count\": len(risk_items)})\n weights = {\"high\": 25, \"medium\": 10, \"low\": 5}\n score = min(\n 100, sum(weights.get(item.get(\"severity\", \"low\"), 5) for item in risk_items)\n )\n level = \"low\" if score < 25 else \"medium\" if score < 60 else \"high\"\n result = {\"score\": score, \"level\": level, \"items\": risk_items}\n log.reflect(\"Risk score computed\", payload={\"score\": score, \"level\": level})\n return result\n\n\n# [/DEF:score_risks:Function]\n\n\n# [/DEF:RiskAssessorModule:Module]\n" }, { "contract_id": "index_by_uuid", "contract_type": "Function", "file_path": "backend/src/core/migration/risk_assessor.py", - "start_line": 35, - "end_line": 53, + "start_line": 38, + "end_line": 56, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -36897,14 +37653,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:index_by_uuid:Function]\n# @PURPOSE: Build UUID-index from normalized objects.\n# @PRE: Input list items are dict-like payloads potentially containing \"uuid\".\n# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]\ndef index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:\n with belief_scope(\"risk_assessor.index_by_uuid\"):\n logger.reason(\"Building UUID index\", extra={\"objects_count\": len(objects)})\n indexed: Dict[str, Dict[str, Any]] = {}\n for obj in objects:\n uuid = obj.get(\"uuid\")\n if uuid:\n indexed[str(uuid)] = obj\n logger.reflect(\"UUID index built\", extra={\"indexed_count\": len(indexed)})\n return indexed\n\n\n# [/DEF:index_by_uuid:Function]\n" + "body": "# [DEF:index_by_uuid:Function]\n# @PURPOSE: Build UUID-index from normalized objects.\n# @PRE: Input list items are dict-like payloads potentially containing \"uuid\".\n# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]\ndef index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:\n with belief_scope(\"risk_assessor.index_by_uuid\"):\n log.reason(\"Building UUID index\", payload={\"objects_count\": len(objects)})\n indexed: Dict[str, Dict[str, Any]] = {}\n for obj in objects:\n uuid = obj.get(\"uuid\")\n if uuid:\n indexed[str(uuid)] = obj\n log.reflect(\"UUID index built\", payload={\"indexed_count\": len(indexed)})\n return indexed\n\n\n# [/DEF:index_by_uuid:Function]\n" }, { "contract_id": "extract_owner_identifiers", "contract_type": "Function", "file_path": "backend/src/core/migration/risk_assessor.py", - "start_line": 56, - "end_line": 84, + "start_line": 59, + "end_line": 87, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -36954,14 +37710,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:extract_owner_identifiers:Function]\n# @PURPOSE: Normalize owner payloads for stable comparison.\n# @PRE: Owners may be list payload, scalar values, or None.\n# @POST: Returns sorted unique owner identifiers as strings.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: Any -> List[str]\ndef extract_owner_identifiers(owners: Any) -> List[str]:\n with belief_scope(\"risk_assessor.extract_owner_identifiers\"):\n logger.reason(\"Normalizing owner identifiers\")\n if not isinstance(owners, list):\n logger.reflect(\"Owners payload is not list; returning empty identifiers\")\n return []\n ids: List[str] = []\n for owner in owners:\n if isinstance(owner, dict):\n if owner.get(\"username\"):\n ids.append(str(owner[\"username\"]))\n elif owner.get(\"id\") is not None:\n ids.append(str(owner[\"id\"]))\n elif owner is not None:\n ids.append(str(owner))\n normalized_ids = sorted(set(ids))\n logger.reflect(\n \"Owner identifiers normalized\", extra={\"owner_count\": len(normalized_ids)}\n )\n return normalized_ids\n\n\n# [/DEF:extract_owner_identifiers:Function]\n" + "body": "# [DEF:extract_owner_identifiers:Function]\n# @PURPOSE: Normalize owner payloads for stable comparison.\n# @PRE: Owners may be list payload, scalar values, or None.\n# @POST: Returns sorted unique owner identifiers as strings.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: Any -> List[str]\ndef extract_owner_identifiers(owners: Any) -> List[str]:\n with belief_scope(\"risk_assessor.extract_owner_identifiers\"):\n log.reason(\"Normalizing owner identifiers\")\n if not isinstance(owners, list):\n log.reflect(\"Owners payload is not list; returning empty identifiers\")\n return []\n ids: List[str] = []\n for owner in owners:\n if isinstance(owner, dict):\n if owner.get(\"username\"):\n ids.append(str(owner[\"username\"]))\n elif owner.get(\"id\") is not None:\n ids.append(str(owner[\"id\"]))\n elif owner is not None:\n ids.append(str(owner))\n normalized_ids = sorted(set(ids))\n log.reflect(\n \"Owner identifiers normalized\", payload={\"owner_count\": len(normalized_ids)}\n )\n return normalized_ids\n\n\n# [/DEF:extract_owner_identifiers:Function]\n" }, { "contract_id": "build_risks", "contract_type": "Function", "file_path": "backend/src/core/migration/risk_assessor.py", - "start_line": 87, - "end_line": 177, + "start_line": 90, + "end_line": 180, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -37033,14 +37789,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:build_risks:Function]\n# @PURPOSE: Build risk list from computed diffs and target catalog state.\n# @RELATION: DEPENDS_ON -> [index_by_uuid]\n# @RELATION: DEPENDS_ON -> [extract_owner_identifiers]\n# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.\n# @PRE: target_client is authenticated/usable for database list retrieval.\n# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.\n# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs.\n# @DATA_CONTRACT: (\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, Dict[str, List[Dict[str, Any]]]],\n# @DATA_CONTRACT: SupersetClient\n# @DATA_CONTRACT: ) -> List[Dict[str, Any]]\ndef build_risks(\n source_objects: Dict[str, List[Dict[str, Any]]],\n target_objects: Dict[str, List[Dict[str, Any]]],\n diff: Dict[str, Dict[str, List[Dict[str, Any]]]],\n target_client: SupersetClient,\n) -> List[Dict[str, Any]]:\n with belief_scope(\"risk_assessor.build_risks\"):\n logger.reason(\"Building migration risks from diff payload\")\n risks: List[Dict[str, Any]] = []\n for object_type in (\"dashboards\", \"charts\", \"datasets\"):\n for item in diff[object_type][\"update\"]:\n risks.append(\n {\n \"code\": \"overwrite_existing\",\n \"severity\": \"medium\",\n \"object_type\": object_type[:-1],\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Object will be updated in target: {item.get('title') or item['uuid']}\",\n }\n )\n\n target_dataset_uuids = set(index_by_uuid(target_objects[\"datasets\"]).keys())\n _, target_databases = target_client.get_databases(query={\"columns\": [\"uuid\"]})\n target_database_uuids = {\n str(item.get(\"uuid\")) for item in target_databases if item.get(\"uuid\")\n }\n\n for dataset in source_objects[\"datasets\"]:\n db_uuid = dataset.get(\"database_uuid\")\n if db_uuid and str(db_uuid) not in target_database_uuids:\n risks.append(\n {\n \"code\": \"missing_datasource\",\n \"severity\": \"high\",\n \"object_type\": \"dataset\",\n \"object_uuid\": dataset.get(\"uuid\"),\n \"message\": f\"Target datasource is missing for dataset {dataset.get('title') or dataset.get('uuid')}\",\n }\n )\n\n for chart in source_objects[\"charts\"]:\n ds_uuid = chart.get(\"dataset_uuid\")\n if ds_uuid and str(ds_uuid) not in target_dataset_uuids:\n risks.append(\n {\n \"code\": \"breaking_reference\",\n \"severity\": \"high\",\n \"object_type\": \"chart\",\n \"object_uuid\": chart.get(\"uuid\"),\n \"message\": f\"Chart references dataset not found on target: {ds_uuid}\",\n }\n )\n\n source_dash = index_by_uuid(source_objects[\"dashboards\"])\n target_dash = index_by_uuid(target_objects[\"dashboards\"])\n for item in diff[\"dashboards\"][\"update\"]:\n source_obj = source_dash.get(item[\"uuid\"])\n target_obj = target_dash.get(item[\"uuid\"])\n if not source_obj or not target_obj:\n continue\n source_owners = extract_owner_identifiers(source_obj.get(\"owners\"))\n target_owners = extract_owner_identifiers(target_obj.get(\"owners\"))\n if source_owners and target_owners and source_owners != target_owners:\n risks.append(\n {\n \"code\": \"owner_mismatch\",\n \"severity\": \"low\",\n \"object_type\": \"dashboard\",\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Owner mismatch for dashboard {item.get('title') or item['uuid']}\",\n }\n )\n logger.reflect(\"Risk list assembled\", extra={\"risk_count\": len(risks)})\n return risks\n\n\n# [/DEF:build_risks:Function]\n" + "body": "# [DEF:build_risks:Function]\n# @PURPOSE: Build risk list from computed diffs and target catalog state.\n# @RELATION: DEPENDS_ON -> [index_by_uuid]\n# @RELATION: DEPENDS_ON -> [extract_owner_identifiers]\n# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.\n# @PRE: target_client is authenticated/usable for database list retrieval.\n# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.\n# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs.\n# @DATA_CONTRACT: (\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],\n# @DATA_CONTRACT: Dict[str, Dict[str, List[Dict[str, Any]]]],\n# @DATA_CONTRACT: SupersetClient\n# @DATA_CONTRACT: ) -> List[Dict[str, Any]]\ndef build_risks(\n source_objects: Dict[str, List[Dict[str, Any]]],\n target_objects: Dict[str, List[Dict[str, Any]]],\n diff: Dict[str, Dict[str, List[Dict[str, Any]]]],\n target_client: SupersetClient,\n) -> List[Dict[str, Any]]:\n with belief_scope(\"risk_assessor.build_risks\"):\n log.reason(\"Building migration risks from diff payload\")\n risks: List[Dict[str, Any]] = []\n for object_type in (\"dashboards\", \"charts\", \"datasets\"):\n for item in diff[object_type][\"update\"]:\n risks.append(\n {\n \"code\": \"overwrite_existing\",\n \"severity\": \"medium\",\n \"object_type\": object_type[:-1],\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Object will be updated in target: {item.get('title') or item['uuid']}\",\n }\n )\n\n target_dataset_uuids = set(index_by_uuid(target_objects[\"datasets\"]).keys())\n _, target_databases = target_client.get_databases(query={\"columns\": [\"uuid\"]})\n target_database_uuids = {\n str(item.get(\"uuid\")) for item in target_databases if item.get(\"uuid\")\n }\n\n for dataset in source_objects[\"datasets\"]:\n db_uuid = dataset.get(\"database_uuid\")\n if db_uuid and str(db_uuid) not in target_database_uuids:\n risks.append(\n {\n \"code\": \"missing_datasource\",\n \"severity\": \"high\",\n \"object_type\": \"dataset\",\n \"object_uuid\": dataset.get(\"uuid\"),\n \"message\": f\"Target datasource is missing for dataset {dataset.get('title') or dataset.get('uuid')}\",\n }\n )\n\n for chart in source_objects[\"charts\"]:\n ds_uuid = chart.get(\"dataset_uuid\")\n if ds_uuid and str(ds_uuid) not in target_dataset_uuids:\n risks.append(\n {\n \"code\": \"breaking_reference\",\n \"severity\": \"high\",\n \"object_type\": \"chart\",\n \"object_uuid\": chart.get(\"uuid\"),\n \"message\": f\"Chart references dataset not found on target: {ds_uuid}\",\n }\n )\n\n source_dash = index_by_uuid(source_objects[\"dashboards\"])\n target_dash = index_by_uuid(target_objects[\"dashboards\"])\n for item in diff[\"dashboards\"][\"update\"]:\n source_obj = source_dash.get(item[\"uuid\"])\n target_obj = target_dash.get(item[\"uuid\"])\n if not source_obj or not target_obj:\n continue\n source_owners = extract_owner_identifiers(source_obj.get(\"owners\"))\n target_owners = extract_owner_identifiers(target_obj.get(\"owners\"))\n if source_owners and target_owners and source_owners != target_owners:\n risks.append(\n {\n \"code\": \"owner_mismatch\",\n \"severity\": \"low\",\n \"object_type\": \"dashboard\",\n \"object_uuid\": item[\"uuid\"],\n \"message\": f\"Owner mismatch for dashboard {item.get('title') or item['uuid']}\",\n }\n )\n log.reflect(\"Risk list assembled\", payload={\"risk_count\": len(risks)})\n return risks\n\n\n# [/DEF:build_risks:Function]\n" }, { "contract_id": "score_risks", "contract_type": "Function", "file_path": "backend/src/core/migration/risk_assessor.py", - "start_line": 180, - "end_line": 199, + "start_line": 183, + "end_line": 202, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -37090,7 +37846,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:score_risks:Function]\n# @PURPOSE: Aggregate risk list into score and level.\n# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.\n# @POST: Returns dict with score in [0,100], derived level, and original items.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any]\ndef score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope(\"risk_assessor.score_risks\"):\n logger.reason(\"Scoring risk items\", extra={\"risk_items_count\": len(risk_items)})\n weights = {\"high\": 25, \"medium\": 10, \"low\": 5}\n score = min(\n 100, sum(weights.get(item.get(\"severity\", \"low\"), 5) for item in risk_items)\n )\n level = \"low\" if score < 25 else \"medium\" if score < 60 else \"high\"\n result = {\"score\": score, \"level\": level, \"items\": risk_items}\n logger.reflect(\"Risk score computed\", extra={\"score\": score, \"level\": level})\n return result\n\n\n# [/DEF:score_risks:Function]\n" + "body": "# [DEF:score_risks:Function]\n# @PURPOSE: Aggregate risk list into score and level.\n# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.\n# @POST: Returns dict with score in [0,100], derived level, and original items.\n# @SIDE_EFFECT: Emits reasoning/reflective logs only.\n# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any]\ndef score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope(\"risk_assessor.score_risks\"):\n log.reason(\"Scoring risk items\", payload={\"risk_items_count\": len(risk_items)})\n weights = {\"high\": 25, \"medium\": 10, \"low\": 5}\n score = min(\n 100, sum(weights.get(item.get(\"severity\", \"low\"), 5) for item in risk_items)\n )\n level = \"low\" if score < 25 else \"medium\" if score < 60 else \"high\"\n result = {\"score\": score, \"level\": level, \"items\": risk_items}\n log.reflect(\"Risk score computed\", payload={\"score\": score, \"level\": level})\n return result\n\n\n# [/DEF:score_risks:Function]\n" }, { "contract_id": "MigrationEngineModule", @@ -38466,7 +39222,7 @@ "contract_type": "Module", "file_path": "backend/src/core/scheduler.py", "start_line": 1, - "end_line": 211, + "end_line": 215, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -38505,14 +39261,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SchedulerModule:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: scheduler, apscheduler, cron, backup\n# @PURPOSE: Manages scheduled tasks using APScheduler.\n# @LAYER: Core\n# @RELATION: DEPENDS_ON -> TaskManager\n\n# [SECTION: IMPORTS]\nfrom apscheduler.schedulers.background import BackgroundScheduler\nfrom apscheduler.triggers.cron import CronTrigger\nfrom .logger import logger, belief_scope\nfrom .config_manager import ConfigManager\nimport asyncio\nfrom datetime import datetime, time, timedelta, date\n# [/SECTION]\n\n\n# [DEF:SchedulerService:Class]\n# @COMPLEXITY: 3\n# @SEMANTICS: scheduler, service, apscheduler\n# @PURPOSE: Provides a service to manage scheduled backup tasks.\n# @RELATION: DEPENDS_ON -> [ThrottledSchedulerConfigurator]\nclass SchedulerService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the scheduler service with task and config managers.\n # @PRE: task_manager and config_manager must be provided.\n # @POST: Scheduler instance is created but not started.\n def __init__(self, task_manager, config_manager: ConfigManager):\n with belief_scope(\"SchedulerService.__init__\"):\n self.task_manager = task_manager\n self.config_manager = config_manager\n self.scheduler = BackgroundScheduler()\n self.loop = asyncio.get_event_loop()\n\n # [/DEF:__init__:Function]\n\n # [DEF:start:Function]\n # @PURPOSE: Starts the background scheduler and loads initial schedules.\n # @PRE: Scheduler should be initialized.\n # @POST: Scheduler is running and schedules are loaded.\n def start(self):\n with belief_scope(\"SchedulerService.start\"):\n if not self.scheduler.running:\n self.scheduler.start()\n logger.info(\"Scheduler started.\")\n self.load_schedules()\n\n # [/DEF:start:Function]\n\n # [DEF:stop:Function]\n # @PURPOSE: Stops the background scheduler.\n # @PRE: Scheduler should be running.\n # @POST: Scheduler is shut down.\n def stop(self):\n with belief_scope(\"SchedulerService.stop\"):\n if self.scheduler.running:\n self.scheduler.shutdown()\n logger.info(\"Scheduler stopped.\")\n\n # [/DEF:stop:Function]\n\n # [DEF:load_schedules:Function]\n # @PURPOSE: Loads backup schedules from configuration and registers them.\n # @PRE: config_manager must have valid configuration.\n # @POST: All enabled backup jobs are added to the scheduler.\n def load_schedules(self):\n with belief_scope(\"SchedulerService.load_schedules\"):\n # Clear existing jobs\n self.scheduler.remove_all_jobs()\n\n config = self.config_manager.get_config()\n for env in config.environments:\n if env.backup_schedule and env.backup_schedule.enabled:\n self.add_backup_job(env.id, env.backup_schedule.cron_expression)\n\n # [/DEF:load_schedules:Function]\n\n # [DEF:add_backup_job:Function]\n # @PURPOSE: Adds a scheduled backup job for an environment.\n # @PRE: env_id and cron_expression must be valid strings.\n # @POST: A new job is added to the scheduler or replaced if it already exists.\n # @PARAM: env_id (str) - The ID of the environment.\n # @PARAM: cron_expression (str) - The cron expression for the schedule.\n def add_backup_job(self, env_id: str, cron_expression: str):\n with belief_scope(\n \"SchedulerService.add_backup_job\",\n f\"env_id={env_id}, cron={cron_expression}\",\n ):\n job_id = f\"backup_{env_id}\"\n try:\n self.scheduler.add_job(\n self._trigger_backup,\n CronTrigger.from_crontab(cron_expression),\n id=job_id,\n args=[env_id],\n replace_existing=True,\n )\n logger.info(\n f\"Scheduled backup job added for environment {env_id}: {cron_expression}\"\n )\n except Exception as e:\n logger.error(f\"Failed to add backup job for environment {env_id}: {e}\")\n\n # [/DEF:add_backup_job:Function]\n\n # [DEF:_trigger_backup:Function]\n # @PURPOSE: Triggered by the scheduler to start a backup task.\n # @PRE: env_id must be a valid environment ID.\n # @POST: A new backup task is created in the task manager if not already running.\n # @PARAM: env_id (str) - The ID of the environment.\n def _trigger_backup(self, env_id: str):\n with belief_scope(\"SchedulerService._trigger_backup\", f\"env_id={env_id}\"):\n logger.info(f\"Triggering scheduled backup for environment {env_id}\")\n\n # Check if a backup is already running for this environment\n active_tasks = self.task_manager.get_tasks(limit=100)\n for task in active_tasks:\n if (\n task.plugin_id == \"superset-backup\"\n and task.status in [\"PENDING\", \"RUNNING\"]\n and task.params.get(\"environment_id\") == env_id\n ):\n logger.warning(\n f\"Backup already running for environment {env_id}. Skipping scheduled run.\"\n )\n return\n\n # Run the backup task\n # We need to run this in the event loop since create_task is async\n asyncio.run_coroutine_threadsafe(\n self.task_manager.create_task(\n \"superset-backup\", {\"environment_id\": env_id}\n ),\n self.loop,\n )\n\n # [/DEF:_trigger_backup:Function]\n\n\n# [/DEF:SchedulerService:Class]\n\n\n# [DEF:ThrottledSchedulerConfigurator:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: scheduler, throttling, distribution\n# @PURPOSE: Distributes validation tasks evenly within an execution window.\n# @PRE: Validation policies provide a finite dashboard list and a valid execution window.\n# @POST: Produces deterministic per-dashboard run timestamps within the configured window.\n# @RELATION: DEPENDS_ON -> SchedulerModule\n# @INVARIANT: Returned schedule size always matches number of dashboard IDs.\n# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.\n# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]\nclass ThrottledSchedulerConfigurator:\n # [DEF:calculate_schedule:Function]\n # @PURPOSE: Calculates execution times for N tasks within a window.\n # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).\n # @POST: Returns List[datetime] of scheduled times.\n # @INVARIANT: Tasks are distributed with near-even spacing.\n @staticmethod\n def calculate_schedule(\n window_start: time, window_end: time, dashboard_ids: list, current_date: date\n ) -> list:\n with belief_scope(\"ThrottledSchedulerConfigurator.calculate_schedule\"):\n n = len(dashboard_ids)\n if n == 0:\n return []\n\n start_dt = datetime.combine(current_date, window_start)\n end_dt = datetime.combine(current_date, window_end)\n\n # Handle window crossing midnight\n if end_dt < start_dt:\n end_dt += timedelta(days=1)\n\n total_seconds = (end_dt - start_dt).total_seconds()\n\n # Minimum interval of 1 second to avoid division by zero or negative\n if total_seconds <= 0:\n logger.warning(\n f\"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks.\"\n )\n return [start_dt] * n\n\n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds),\n # we still distribute them but they might be very close.\n # The requirement says \"near-even spacing\".\n\n if n == 1:\n return [start_dt]\n\n interval = total_seconds / (n - 1) if n > 1 else 0\n\n # If interval is too small (e.g. < 1s), we might want a fallback,\n # but the spec says \"handle too-small windows with explicit fallback/warning\".\n if interval < 1:\n logger.warning(\n f\"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated.\"\n )\n\n scheduled_times = []\n for i in range(n):\n scheduled_times.append(start_dt + timedelta(seconds=i * interval))\n\n return scheduled_times\n\n # [/DEF:calculate_schedule:Function]\n\n\n# [/DEF:ThrottledSchedulerConfigurator:Class]\n\n# [/DEF:SchedulerModule:Module]\n" + "body": "# [DEF:SchedulerModule:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: scheduler, apscheduler, cron, backup\n# @PURPOSE: Manages scheduled tasks using APScheduler.\n# @LAYER: Core\n# @RELATION: DEPENDS_ON -> TaskManager\n\n# [SECTION: IMPORTS]\nfrom apscheduler.schedulers.background import BackgroundScheduler\nfrom apscheduler.triggers.cron import CronTrigger\nfrom .logger import logger, belief_scope\nfrom .config_manager import ConfigManager\nfrom .cot_logger import MarkerLogger, get_trace_id, set_trace_id\n\nlog = MarkerLogger(\"SchedulerService\")\nlog_tsc = MarkerLogger(\"ThrottledSchedulerConfigurator\")\nimport asyncio\nfrom datetime import datetime, time, timedelta, date\n# [/SECTION]\n\n\n# [DEF:SchedulerService:Class]\n# @COMPLEXITY: 3\n# @SEMANTICS: scheduler, service, apscheduler\n# @PURPOSE: Provides a service to manage scheduled backup tasks.\n# @RELATION: DEPENDS_ON -> [ThrottledSchedulerConfigurator]\nclass SchedulerService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the scheduler service with task and config managers.\n # @PRE: task_manager and config_manager must be provided.\n # @POST: Scheduler instance is created but not started.\n def __init__(self, task_manager, config_manager: ConfigManager):\n with belief_scope(\"SchedulerService.__init__\"):\n self.task_manager = task_manager\n self.config_manager = config_manager\n self.scheduler = BackgroundScheduler()\n self.loop = asyncio.get_event_loop()\n\n # [/DEF:__init__:Function]\n\n # [DEF:start:Function]\n # @PURPOSE: Starts the background scheduler and loads initial schedules.\n # @PRE: Scheduler should be initialized.\n # @POST: Scheduler is running and schedules are loaded.\n def start(self):\n with belief_scope(\"SchedulerService.start\"):\n if not self.scheduler.running:\n self.scheduler.start()\n log.reflect(\"Scheduler started\")\n self.load_schedules()\n\n # [/DEF:start:Function]\n\n # [DEF:stop:Function]\n # @PURPOSE: Stops the background scheduler.\n # @PRE: Scheduler should be running.\n # @POST: Scheduler is shut down.\n def stop(self):\n with belief_scope(\"SchedulerService.stop\"):\n if self.scheduler.running:\n self.scheduler.shutdown()\n log.reflect(\"Scheduler stopped\")\n\n # [/DEF:stop:Function]\n\n # [DEF:load_schedules:Function]\n # @PURPOSE: Loads backup schedules from configuration and registers them.\n # @PRE: config_manager must have valid configuration.\n # @POST: All enabled backup jobs are added to the scheduler.\n def load_schedules(self):\n with belief_scope(\"SchedulerService.load_schedules\"):\n # Clear existing jobs\n self.scheduler.remove_all_jobs()\n\n config = self.config_manager.get_config()\n for env in config.environments:\n if env.backup_schedule and env.backup_schedule.enabled:\n self.add_backup_job(env.id, env.backup_schedule.cron_expression)\n\n # [/DEF:load_schedules:Function]\n\n # [DEF:add_backup_job:Function]\n # @PURPOSE: Adds a scheduled backup job for an environment.\n # @PRE: env_id and cron_expression must be valid strings.\n # @POST: A new job is added to the scheduler or replaced if it already exists.\n # @PARAM: env_id (str) - The ID of the environment.\n # @PARAM: cron_expression (str) - The cron expression for the schedule.\n def add_backup_job(self, env_id: str, cron_expression: str):\n with belief_scope(\n \"SchedulerService.add_backup_job\",\n f\"env_id={env_id}, cron={cron_expression}\",\n ):\n job_id = f\"backup_{env_id}\"\n try:\n self.scheduler.add_job(\n self._trigger_backup,\n CronTrigger.from_crontab(cron_expression),\n id=job_id,\n args=[env_id],\n replace_existing=True,\n )\n log.reflect(\n \"Scheduled backup job added\",\n payload={\"env_id\": env_id, \"cron\": cron_expression},\n )\n except Exception as e:\n log.explore(\"Failed to add backup job\", error=str(e), payload={\"env_id\": env_id, \"cron\": cron_expression})\n logger.error(f\"Failed to add backup job for environment {env_id}: {e}\")\n\n # [/DEF:add_backup_job:Function]\n\n # [DEF:_trigger_backup:Function]\n # @PURPOSE: Triggered by the scheduler to start a backup task.\n # @PRE: env_id must be a valid environment ID.\n # @POST: A new backup task is created in the task manager if not already running.\n # @PARAM: env_id (str) - The ID of the environment.\n def _trigger_backup(self, env_id: str):\n with belief_scope(\"SchedulerService._trigger_backup\", f\"env_id={env_id}\"):\n log.reason(\"Triggering scheduled backup\", payload={\"env_id\": env_id})\n\n # Check if a backup is already running for this environment\n active_tasks = self.task_manager.get_tasks(limit=100)\n for task in active_tasks:\n if (\n task.plugin_id == \"superset-backup\"\n and task.status in [\"PENDING\", \"RUNNING\"]\n and task.params.get(\"environment_id\") == env_id\n ):\n log.explore(\"Backup already running, skipping scheduled run\", error=\"Duplicate backup prevented\", payload={\"env_id\": env_id})\n return\n\n # Run the backup task\n # We need to run this in the event loop since create_task is async\n trace_id = get_trace_id()\n\n async def _backup_task():\n if trace_id:\n set_trace_id(trace_id)\n await self.task_manager.create_task(\n \"superset-backup\", {\"environment_id\": env_id}\n )\n\n asyncio.run_coroutine_threadsafe(_backup_task(), self.loop)\n\n # [/DEF:_trigger_backup:Function]\n\n\n# [/DEF:SchedulerService:Class]\n\n\n# [DEF:ThrottledSchedulerConfigurator:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: scheduler, throttling, distribution\n# @PURPOSE: Distributes validation tasks evenly within an execution window.\n# @PRE: Validation policies provide a finite dashboard list and a valid execution window.\n# @POST: Produces deterministic per-dashboard run timestamps within the configured window.\n# @RELATION: DEPENDS_ON -> SchedulerModule\n# @INVARIANT: Returned schedule size always matches number of dashboard IDs.\n# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.\n# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]\nclass ThrottledSchedulerConfigurator:\n # [DEF:calculate_schedule:Function]\n # @PURPOSE: Calculates execution times for N tasks within a window.\n # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).\n # @POST: Returns List[datetime] of scheduled times.\n # @INVARIANT: Tasks are distributed with near-even spacing.\n @staticmethod\n def calculate_schedule(\n window_start: time, window_end: time, dashboard_ids: list, current_date: date\n ) -> list:\n with belief_scope(\"ThrottledSchedulerConfigurator.calculate_schedule\"):\n n = len(dashboard_ids)\n if n == 0:\n return []\n\n start_dt = datetime.combine(current_date, window_start)\n end_dt = datetime.combine(current_date, window_end)\n\n # Handle window crossing midnight\n if end_dt < start_dt:\n end_dt += timedelta(days=1)\n\n total_seconds = (end_dt - start_dt).total_seconds()\n\n # Minimum interval of 1 second to avoid division by zero or negative\n if total_seconds <= 0:\n log_tsc.explore(\"Window size is zero or negative, falling back to start time\", error=f\"total_seconds={total_seconds}\", payload={\"n\": n})\n return [start_dt] * n\n\n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds),\n # we still distribute them but they might be very close.\n # The requirement says \"near-even spacing\".\n\n if n == 1:\n return [start_dt]\n\n interval = total_seconds / (n - 1) if n > 1 else 0\n\n # If interval is too small (e.g. < 1s), we might want a fallback,\n # but the spec says \"handle too-small windows with explicit fallback/warning\".\n if interval < 1:\n log_tsc.explore(\"Window too small for task distribution\", error=f\"interval={interval:.2f}s\", payload={\"n\": n})\n\n scheduled_times = []\n for i in range(n):\n scheduled_times.append(start_dt + timedelta(seconds=i * interval))\n\n return scheduled_times\n\n # [/DEF:calculate_schedule:Function]\n\n\n# [/DEF:ThrottledSchedulerConfigurator:Class]\n\n# [/DEF:SchedulerModule:Module]\n" }, { "contract_id": "SchedulerService", "contract_type": "Class", "file_path": "backend/src/core/scheduler.py", - "start_line": 18, - "end_line": 140, + "start_line": 22, + "end_line": 148, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -38555,14 +39311,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SchedulerService:Class]\n# @COMPLEXITY: 3\n# @SEMANTICS: scheduler, service, apscheduler\n# @PURPOSE: Provides a service to manage scheduled backup tasks.\n# @RELATION: DEPENDS_ON -> [ThrottledSchedulerConfigurator]\nclass SchedulerService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the scheduler service with task and config managers.\n # @PRE: task_manager and config_manager must be provided.\n # @POST: Scheduler instance is created but not started.\n def __init__(self, task_manager, config_manager: ConfigManager):\n with belief_scope(\"SchedulerService.__init__\"):\n self.task_manager = task_manager\n self.config_manager = config_manager\n self.scheduler = BackgroundScheduler()\n self.loop = asyncio.get_event_loop()\n\n # [/DEF:__init__:Function]\n\n # [DEF:start:Function]\n # @PURPOSE: Starts the background scheduler and loads initial schedules.\n # @PRE: Scheduler should be initialized.\n # @POST: Scheduler is running and schedules are loaded.\n def start(self):\n with belief_scope(\"SchedulerService.start\"):\n if not self.scheduler.running:\n self.scheduler.start()\n logger.info(\"Scheduler started.\")\n self.load_schedules()\n\n # [/DEF:start:Function]\n\n # [DEF:stop:Function]\n # @PURPOSE: Stops the background scheduler.\n # @PRE: Scheduler should be running.\n # @POST: Scheduler is shut down.\n def stop(self):\n with belief_scope(\"SchedulerService.stop\"):\n if self.scheduler.running:\n self.scheduler.shutdown()\n logger.info(\"Scheduler stopped.\")\n\n # [/DEF:stop:Function]\n\n # [DEF:load_schedules:Function]\n # @PURPOSE: Loads backup schedules from configuration and registers them.\n # @PRE: config_manager must have valid configuration.\n # @POST: All enabled backup jobs are added to the scheduler.\n def load_schedules(self):\n with belief_scope(\"SchedulerService.load_schedules\"):\n # Clear existing jobs\n self.scheduler.remove_all_jobs()\n\n config = self.config_manager.get_config()\n for env in config.environments:\n if env.backup_schedule and env.backup_schedule.enabled:\n self.add_backup_job(env.id, env.backup_schedule.cron_expression)\n\n # [/DEF:load_schedules:Function]\n\n # [DEF:add_backup_job:Function]\n # @PURPOSE: Adds a scheduled backup job for an environment.\n # @PRE: env_id and cron_expression must be valid strings.\n # @POST: A new job is added to the scheduler or replaced if it already exists.\n # @PARAM: env_id (str) - The ID of the environment.\n # @PARAM: cron_expression (str) - The cron expression for the schedule.\n def add_backup_job(self, env_id: str, cron_expression: str):\n with belief_scope(\n \"SchedulerService.add_backup_job\",\n f\"env_id={env_id}, cron={cron_expression}\",\n ):\n job_id = f\"backup_{env_id}\"\n try:\n self.scheduler.add_job(\n self._trigger_backup,\n CronTrigger.from_crontab(cron_expression),\n id=job_id,\n args=[env_id],\n replace_existing=True,\n )\n logger.info(\n f\"Scheduled backup job added for environment {env_id}: {cron_expression}\"\n )\n except Exception as e:\n logger.error(f\"Failed to add backup job for environment {env_id}: {e}\")\n\n # [/DEF:add_backup_job:Function]\n\n # [DEF:_trigger_backup:Function]\n # @PURPOSE: Triggered by the scheduler to start a backup task.\n # @PRE: env_id must be a valid environment ID.\n # @POST: A new backup task is created in the task manager if not already running.\n # @PARAM: env_id (str) - The ID of the environment.\n def _trigger_backup(self, env_id: str):\n with belief_scope(\"SchedulerService._trigger_backup\", f\"env_id={env_id}\"):\n logger.info(f\"Triggering scheduled backup for environment {env_id}\")\n\n # Check if a backup is already running for this environment\n active_tasks = self.task_manager.get_tasks(limit=100)\n for task in active_tasks:\n if (\n task.plugin_id == \"superset-backup\"\n and task.status in [\"PENDING\", \"RUNNING\"]\n and task.params.get(\"environment_id\") == env_id\n ):\n logger.warning(\n f\"Backup already running for environment {env_id}. Skipping scheduled run.\"\n )\n return\n\n # Run the backup task\n # We need to run this in the event loop since create_task is async\n asyncio.run_coroutine_threadsafe(\n self.task_manager.create_task(\n \"superset-backup\", {\"environment_id\": env_id}\n ),\n self.loop,\n )\n\n # [/DEF:_trigger_backup:Function]\n\n\n# [/DEF:SchedulerService:Class]\n" + "body": "# [DEF:SchedulerService:Class]\n# @COMPLEXITY: 3\n# @SEMANTICS: scheduler, service, apscheduler\n# @PURPOSE: Provides a service to manage scheduled backup tasks.\n# @RELATION: DEPENDS_ON -> [ThrottledSchedulerConfigurator]\nclass SchedulerService:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the scheduler service with task and config managers.\n # @PRE: task_manager and config_manager must be provided.\n # @POST: Scheduler instance is created but not started.\n def __init__(self, task_manager, config_manager: ConfigManager):\n with belief_scope(\"SchedulerService.__init__\"):\n self.task_manager = task_manager\n self.config_manager = config_manager\n self.scheduler = BackgroundScheduler()\n self.loop = asyncio.get_event_loop()\n\n # [/DEF:__init__:Function]\n\n # [DEF:start:Function]\n # @PURPOSE: Starts the background scheduler and loads initial schedules.\n # @PRE: Scheduler should be initialized.\n # @POST: Scheduler is running and schedules are loaded.\n def start(self):\n with belief_scope(\"SchedulerService.start\"):\n if not self.scheduler.running:\n self.scheduler.start()\n log.reflect(\"Scheduler started\")\n self.load_schedules()\n\n # [/DEF:start:Function]\n\n # [DEF:stop:Function]\n # @PURPOSE: Stops the background scheduler.\n # @PRE: Scheduler should be running.\n # @POST: Scheduler is shut down.\n def stop(self):\n with belief_scope(\"SchedulerService.stop\"):\n if self.scheduler.running:\n self.scheduler.shutdown()\n log.reflect(\"Scheduler stopped\")\n\n # [/DEF:stop:Function]\n\n # [DEF:load_schedules:Function]\n # @PURPOSE: Loads backup schedules from configuration and registers them.\n # @PRE: config_manager must have valid configuration.\n # @POST: All enabled backup jobs are added to the scheduler.\n def load_schedules(self):\n with belief_scope(\"SchedulerService.load_schedules\"):\n # Clear existing jobs\n self.scheduler.remove_all_jobs()\n\n config = self.config_manager.get_config()\n for env in config.environments:\n if env.backup_schedule and env.backup_schedule.enabled:\n self.add_backup_job(env.id, env.backup_schedule.cron_expression)\n\n # [/DEF:load_schedules:Function]\n\n # [DEF:add_backup_job:Function]\n # @PURPOSE: Adds a scheduled backup job for an environment.\n # @PRE: env_id and cron_expression must be valid strings.\n # @POST: A new job is added to the scheduler or replaced if it already exists.\n # @PARAM: env_id (str) - The ID of the environment.\n # @PARAM: cron_expression (str) - The cron expression for the schedule.\n def add_backup_job(self, env_id: str, cron_expression: str):\n with belief_scope(\n \"SchedulerService.add_backup_job\",\n f\"env_id={env_id}, cron={cron_expression}\",\n ):\n job_id = f\"backup_{env_id}\"\n try:\n self.scheduler.add_job(\n self._trigger_backup,\n CronTrigger.from_crontab(cron_expression),\n id=job_id,\n args=[env_id],\n replace_existing=True,\n )\n log.reflect(\n \"Scheduled backup job added\",\n payload={\"env_id\": env_id, \"cron\": cron_expression},\n )\n except Exception as e:\n log.explore(\"Failed to add backup job\", error=str(e), payload={\"env_id\": env_id, \"cron\": cron_expression})\n logger.error(f\"Failed to add backup job for environment {env_id}: {e}\")\n\n # [/DEF:add_backup_job:Function]\n\n # [DEF:_trigger_backup:Function]\n # @PURPOSE: Triggered by the scheduler to start a backup task.\n # @PRE: env_id must be a valid environment ID.\n # @POST: A new backup task is created in the task manager if not already running.\n # @PARAM: env_id (str) - The ID of the environment.\n def _trigger_backup(self, env_id: str):\n with belief_scope(\"SchedulerService._trigger_backup\", f\"env_id={env_id}\"):\n log.reason(\"Triggering scheduled backup\", payload={\"env_id\": env_id})\n\n # Check if a backup is already running for this environment\n active_tasks = self.task_manager.get_tasks(limit=100)\n for task in active_tasks:\n if (\n task.plugin_id == \"superset-backup\"\n and task.status in [\"PENDING\", \"RUNNING\"]\n and task.params.get(\"environment_id\") == env_id\n ):\n log.explore(\"Backup already running, skipping scheduled run\", error=\"Duplicate backup prevented\", payload={\"env_id\": env_id})\n return\n\n # Run the backup task\n # We need to run this in the event loop since create_task is async\n trace_id = get_trace_id()\n\n async def _backup_task():\n if trace_id:\n set_trace_id(trace_id)\n await self.task_manager.create_task(\n \"superset-backup\", {\"environment_id\": env_id}\n )\n\n asyncio.run_coroutine_threadsafe(_backup_task(), self.loop)\n\n # [/DEF:_trigger_backup:Function]\n\n\n# [/DEF:SchedulerService:Class]\n" }, { "contract_id": "start", "contract_type": "Function", "file_path": "backend/src/core/scheduler.py", - "start_line": 37, - "end_line": 48, + "start_line": 41, + "end_line": 52, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -38592,14 +39348,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:start:Function]\n # @PURPOSE: Starts the background scheduler and loads initial schedules.\n # @PRE: Scheduler should be initialized.\n # @POST: Scheduler is running and schedules are loaded.\n def start(self):\n with belief_scope(\"SchedulerService.start\"):\n if not self.scheduler.running:\n self.scheduler.start()\n logger.info(\"Scheduler started.\")\n self.load_schedules()\n\n # [/DEF:start:Function]\n" + "body": " # [DEF:start:Function]\n # @PURPOSE: Starts the background scheduler and loads initial schedules.\n # @PRE: Scheduler should be initialized.\n # @POST: Scheduler is running and schedules are loaded.\n def start(self):\n with belief_scope(\"SchedulerService.start\"):\n if not self.scheduler.running:\n self.scheduler.start()\n log.reflect(\"Scheduler started\")\n self.load_schedules()\n\n # [/DEF:start:Function]\n" }, { "contract_id": "stop", "contract_type": "Function", "file_path": "backend/src/core/scheduler.py", - "start_line": 50, - "end_line": 60, + "start_line": 54, + "end_line": 64, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -38629,14 +39385,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:stop:Function]\n # @PURPOSE: Stops the background scheduler.\n # @PRE: Scheduler should be running.\n # @POST: Scheduler is shut down.\n def stop(self):\n with belief_scope(\"SchedulerService.stop\"):\n if self.scheduler.running:\n self.scheduler.shutdown()\n logger.info(\"Scheduler stopped.\")\n\n # [/DEF:stop:Function]\n" + "body": " # [DEF:stop:Function]\n # @PURPOSE: Stops the background scheduler.\n # @PRE: Scheduler should be running.\n # @POST: Scheduler is shut down.\n def stop(self):\n with belief_scope(\"SchedulerService.stop\"):\n if self.scheduler.running:\n self.scheduler.shutdown()\n log.reflect(\"Scheduler stopped\")\n\n # [/DEF:stop:Function]\n" }, { "contract_id": "load_schedules", "contract_type": "Function", "file_path": "backend/src/core/scheduler.py", - "start_line": 62, - "end_line": 76, + "start_line": 66, + "end_line": 80, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -38672,8 +39428,8 @@ "contract_id": "add_backup_job", "contract_type": "Function", "file_path": "backend/src/core/scheduler.py", - "start_line": 78, - "end_line": 104, + "start_line": 82, + "end_line": 110, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -38710,14 +39466,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:add_backup_job:Function]\n # @PURPOSE: Adds a scheduled backup job for an environment.\n # @PRE: env_id and cron_expression must be valid strings.\n # @POST: A new job is added to the scheduler or replaced if it already exists.\n # @PARAM: env_id (str) - The ID of the environment.\n # @PARAM: cron_expression (str) - The cron expression for the schedule.\n def add_backup_job(self, env_id: str, cron_expression: str):\n with belief_scope(\n \"SchedulerService.add_backup_job\",\n f\"env_id={env_id}, cron={cron_expression}\",\n ):\n job_id = f\"backup_{env_id}\"\n try:\n self.scheduler.add_job(\n self._trigger_backup,\n CronTrigger.from_crontab(cron_expression),\n id=job_id,\n args=[env_id],\n replace_existing=True,\n )\n logger.info(\n f\"Scheduled backup job added for environment {env_id}: {cron_expression}\"\n )\n except Exception as e:\n logger.error(f\"Failed to add backup job for environment {env_id}: {e}\")\n\n # [/DEF:add_backup_job:Function]\n" + "body": " # [DEF:add_backup_job:Function]\n # @PURPOSE: Adds a scheduled backup job for an environment.\n # @PRE: env_id and cron_expression must be valid strings.\n # @POST: A new job is added to the scheduler or replaced if it already exists.\n # @PARAM: env_id (str) - The ID of the environment.\n # @PARAM: cron_expression (str) - The cron expression for the schedule.\n def add_backup_job(self, env_id: str, cron_expression: str):\n with belief_scope(\n \"SchedulerService.add_backup_job\",\n f\"env_id={env_id}, cron={cron_expression}\",\n ):\n job_id = f\"backup_{env_id}\"\n try:\n self.scheduler.add_job(\n self._trigger_backup,\n CronTrigger.from_crontab(cron_expression),\n id=job_id,\n args=[env_id],\n replace_existing=True,\n )\n log.reflect(\n \"Scheduled backup job added\",\n payload={\"env_id\": env_id, \"cron\": cron_expression},\n )\n except Exception as e:\n log.explore(\"Failed to add backup job\", error=str(e), payload={\"env_id\": env_id, \"cron\": cron_expression})\n logger.error(f\"Failed to add backup job for environment {env_id}: {e}\")\n\n # [/DEF:add_backup_job:Function]\n" }, { "contract_id": "_trigger_backup", "contract_type": "Function", "file_path": "backend/src/core/scheduler.py", - "start_line": 106, - "end_line": 137, + "start_line": 112, + "end_line": 145, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -38754,14 +39510,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_trigger_backup:Function]\n # @PURPOSE: Triggered by the scheduler to start a backup task.\n # @PRE: env_id must be a valid environment ID.\n # @POST: A new backup task is created in the task manager if not already running.\n # @PARAM: env_id (str) - The ID of the environment.\n def _trigger_backup(self, env_id: str):\n with belief_scope(\"SchedulerService._trigger_backup\", f\"env_id={env_id}\"):\n logger.info(f\"Triggering scheduled backup for environment {env_id}\")\n\n # Check if a backup is already running for this environment\n active_tasks = self.task_manager.get_tasks(limit=100)\n for task in active_tasks:\n if (\n task.plugin_id == \"superset-backup\"\n and task.status in [\"PENDING\", \"RUNNING\"]\n and task.params.get(\"environment_id\") == env_id\n ):\n logger.warning(\n f\"Backup already running for environment {env_id}. Skipping scheduled run.\"\n )\n return\n\n # Run the backup task\n # We need to run this in the event loop since create_task is async\n asyncio.run_coroutine_threadsafe(\n self.task_manager.create_task(\n \"superset-backup\", {\"environment_id\": env_id}\n ),\n self.loop,\n )\n\n # [/DEF:_trigger_backup:Function]\n" + "body": " # [DEF:_trigger_backup:Function]\n # @PURPOSE: Triggered by the scheduler to start a backup task.\n # @PRE: env_id must be a valid environment ID.\n # @POST: A new backup task is created in the task manager if not already running.\n # @PARAM: env_id (str) - The ID of the environment.\n def _trigger_backup(self, env_id: str):\n with belief_scope(\"SchedulerService._trigger_backup\", f\"env_id={env_id}\"):\n log.reason(\"Triggering scheduled backup\", payload={\"env_id\": env_id})\n\n # Check if a backup is already running for this environment\n active_tasks = self.task_manager.get_tasks(limit=100)\n for task in active_tasks:\n if (\n task.plugin_id == \"superset-backup\"\n and task.status in [\"PENDING\", \"RUNNING\"]\n and task.params.get(\"environment_id\") == env_id\n ):\n log.explore(\"Backup already running, skipping scheduled run\", error=\"Duplicate backup prevented\", payload={\"env_id\": env_id})\n return\n\n # Run the backup task\n # We need to run this in the event loop since create_task is async\n trace_id = get_trace_id()\n\n async def _backup_task():\n if trace_id:\n set_trace_id(trace_id)\n await self.task_manager.create_task(\n \"superset-backup\", {\"environment_id\": env_id}\n )\n\n asyncio.run_coroutine_threadsafe(_backup_task(), self.loop)\n\n # [/DEF:_trigger_backup:Function]\n" }, { "contract_id": "ThrottledSchedulerConfigurator", "contract_type": "Class", "file_path": "backend/src/core/scheduler.py", - "start_line": 143, - "end_line": 209, + "start_line": 151, + "end_line": 213, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -38809,14 +39565,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:ThrottledSchedulerConfigurator:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: scheduler, throttling, distribution\n# @PURPOSE: Distributes validation tasks evenly within an execution window.\n# @PRE: Validation policies provide a finite dashboard list and a valid execution window.\n# @POST: Produces deterministic per-dashboard run timestamps within the configured window.\n# @RELATION: DEPENDS_ON -> SchedulerModule\n# @INVARIANT: Returned schedule size always matches number of dashboard IDs.\n# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.\n# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]\nclass ThrottledSchedulerConfigurator:\n # [DEF:calculate_schedule:Function]\n # @PURPOSE: Calculates execution times for N tasks within a window.\n # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).\n # @POST: Returns List[datetime] of scheduled times.\n # @INVARIANT: Tasks are distributed with near-even spacing.\n @staticmethod\n def calculate_schedule(\n window_start: time, window_end: time, dashboard_ids: list, current_date: date\n ) -> list:\n with belief_scope(\"ThrottledSchedulerConfigurator.calculate_schedule\"):\n n = len(dashboard_ids)\n if n == 0:\n return []\n\n start_dt = datetime.combine(current_date, window_start)\n end_dt = datetime.combine(current_date, window_end)\n\n # Handle window crossing midnight\n if end_dt < start_dt:\n end_dt += timedelta(days=1)\n\n total_seconds = (end_dt - start_dt).total_seconds()\n\n # Minimum interval of 1 second to avoid division by zero or negative\n if total_seconds <= 0:\n logger.warning(\n f\"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks.\"\n )\n return [start_dt] * n\n\n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds),\n # we still distribute them but they might be very close.\n # The requirement says \"near-even spacing\".\n\n if n == 1:\n return [start_dt]\n\n interval = total_seconds / (n - 1) if n > 1 else 0\n\n # If interval is too small (e.g. < 1s), we might want a fallback,\n # but the spec says \"handle too-small windows with explicit fallback/warning\".\n if interval < 1:\n logger.warning(\n f\"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated.\"\n )\n\n scheduled_times = []\n for i in range(n):\n scheduled_times.append(start_dt + timedelta(seconds=i * interval))\n\n return scheduled_times\n\n # [/DEF:calculate_schedule:Function]\n\n\n# [/DEF:ThrottledSchedulerConfigurator:Class]\n" + "body": "# [DEF:ThrottledSchedulerConfigurator:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: scheduler, throttling, distribution\n# @PURPOSE: Distributes validation tasks evenly within an execution window.\n# @PRE: Validation policies provide a finite dashboard list and a valid execution window.\n# @POST: Produces deterministic per-dashboard run timestamps within the configured window.\n# @RELATION: DEPENDS_ON -> SchedulerModule\n# @INVARIANT: Returned schedule size always matches number of dashboard IDs.\n# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.\n# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]\nclass ThrottledSchedulerConfigurator:\n # [DEF:calculate_schedule:Function]\n # @PURPOSE: Calculates execution times for N tasks within a window.\n # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).\n # @POST: Returns List[datetime] of scheduled times.\n # @INVARIANT: Tasks are distributed with near-even spacing.\n @staticmethod\n def calculate_schedule(\n window_start: time, window_end: time, dashboard_ids: list, current_date: date\n ) -> list:\n with belief_scope(\"ThrottledSchedulerConfigurator.calculate_schedule\"):\n n = len(dashboard_ids)\n if n == 0:\n return []\n\n start_dt = datetime.combine(current_date, window_start)\n end_dt = datetime.combine(current_date, window_end)\n\n # Handle window crossing midnight\n if end_dt < start_dt:\n end_dt += timedelta(days=1)\n\n total_seconds = (end_dt - start_dt).total_seconds()\n\n # Minimum interval of 1 second to avoid division by zero or negative\n if total_seconds <= 0:\n log_tsc.explore(\"Window size is zero or negative, falling back to start time\", error=f\"total_seconds={total_seconds}\", payload={\"n\": n})\n return [start_dt] * n\n\n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds),\n # we still distribute them but they might be very close.\n # The requirement says \"near-even spacing\".\n\n if n == 1:\n return [start_dt]\n\n interval = total_seconds / (n - 1) if n > 1 else 0\n\n # If interval is too small (e.g. < 1s), we might want a fallback,\n # but the spec says \"handle too-small windows with explicit fallback/warning\".\n if interval < 1:\n log_tsc.explore(\"Window too small for task distribution\", error=f\"interval={interval:.2f}s\", payload={\"n\": n})\n\n scheduled_times = []\n for i in range(n):\n scheduled_times.append(start_dt + timedelta(seconds=i * interval))\n\n return scheduled_times\n\n # [/DEF:calculate_schedule:Function]\n\n\n# [/DEF:ThrottledSchedulerConfigurator:Class]\n" }, { "contract_id": "calculate_schedule", "contract_type": "Function", "file_path": "backend/src/core/scheduler.py", - "start_line": 154, - "end_line": 206, + "start_line": 162, + "end_line": 210, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -38856,7 +39612,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:calculate_schedule:Function]\n # @PURPOSE: Calculates execution times for N tasks within a window.\n # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).\n # @POST: Returns List[datetime] of scheduled times.\n # @INVARIANT: Tasks are distributed with near-even spacing.\n @staticmethod\n def calculate_schedule(\n window_start: time, window_end: time, dashboard_ids: list, current_date: date\n ) -> list:\n with belief_scope(\"ThrottledSchedulerConfigurator.calculate_schedule\"):\n n = len(dashboard_ids)\n if n == 0:\n return []\n\n start_dt = datetime.combine(current_date, window_start)\n end_dt = datetime.combine(current_date, window_end)\n\n # Handle window crossing midnight\n if end_dt < start_dt:\n end_dt += timedelta(days=1)\n\n total_seconds = (end_dt - start_dt).total_seconds()\n\n # Minimum interval of 1 second to avoid division by zero or negative\n if total_seconds <= 0:\n logger.warning(\n f\"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks.\"\n )\n return [start_dt] * n\n\n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds),\n # we still distribute them but they might be very close.\n # The requirement says \"near-even spacing\".\n\n if n == 1:\n return [start_dt]\n\n interval = total_seconds / (n - 1) if n > 1 else 0\n\n # If interval is too small (e.g. < 1s), we might want a fallback,\n # but the spec says \"handle too-small windows with explicit fallback/warning\".\n if interval < 1:\n logger.warning(\n f\"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated.\"\n )\n\n scheduled_times = []\n for i in range(n):\n scheduled_times.append(start_dt + timedelta(seconds=i * interval))\n\n return scheduled_times\n\n # [/DEF:calculate_schedule:Function]\n" + "body": " # [DEF:calculate_schedule:Function]\n # @PURPOSE: Calculates execution times for N tasks within a window.\n # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).\n # @POST: Returns List[datetime] of scheduled times.\n # @INVARIANT: Tasks are distributed with near-even spacing.\n @staticmethod\n def calculate_schedule(\n window_start: time, window_end: time, dashboard_ids: list, current_date: date\n ) -> list:\n with belief_scope(\"ThrottledSchedulerConfigurator.calculate_schedule\"):\n n = len(dashboard_ids)\n if n == 0:\n return []\n\n start_dt = datetime.combine(current_date, window_start)\n end_dt = datetime.combine(current_date, window_end)\n\n # Handle window crossing midnight\n if end_dt < start_dt:\n end_dt += timedelta(days=1)\n\n total_seconds = (end_dt - start_dt).total_seconds()\n\n # Minimum interval of 1 second to avoid division by zero or negative\n if total_seconds <= 0:\n log_tsc.explore(\"Window size is zero or negative, falling back to start time\", error=f\"total_seconds={total_seconds}\", payload={\"n\": n})\n return [start_dt] * n\n\n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds),\n # we still distribute them but they might be very close.\n # The requirement says \"near-even spacing\".\n\n if n == 1:\n return [start_dt]\n\n interval = total_seconds / (n - 1) if n > 1 else 0\n\n # If interval is too small (e.g. < 1s), we might want a fallback,\n # but the spec says \"handle too-small windows with explicit fallback/warning\".\n if interval < 1:\n log_tsc.explore(\"Window too small for task distribution\", error=f\"interval={interval:.2f}s\", payload={\"n\": n})\n\n scheduled_times = []\n for i in range(n):\n scheduled_times.append(start_dt + timedelta(seconds=i * interval))\n\n return scheduled_times\n\n # [/DEF:calculate_schedule:Function]\n" }, { "contract_id": "SupersetClientModule", @@ -39038,7 +39794,7 @@ "contract_type": "Module", "file_path": "backend/src/core/superset_client/_base.py", "start_line": 1, - "end_line": 315, + "end_line": 300, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39084,14 +39840,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetClientBase:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, api, client, base, pagination, auth, import, export\n# @PURPOSE: Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers.\n# @RELATION: DEPENDS_ON -> [ConfigModels]\n# @RELATION: DEPENDS_ON -> [APIClient]\n# @RELATION: DEPENDS_ON -> [SupersetAPIError]\n# @RELATION: DEPENDS_ON -> [get_filename_from_headers]\n\nimport json\nimport zipfile\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union, cast\nfrom requests import Response\n\nfrom ..logger import logger as app_logger, belief_scope\nfrom ..utils.network import APIClient, SupersetAPIError\nfrom ..utils.fileio import get_filename_from_headers\nfrom ..config_models import Environment\n\napp_logger = cast(Any, app_logger)\n\n\n# [DEF:SupersetClientBase:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Base class providing Superset client initialization, auth, pagination, and import/export plumbing.\n# @RELATION: DEPENDS_ON -> [ConfigModels]\n# @RELATION: DEPENDS_ON -> [APIClient]\n# @RELATION: DEPENDS_ON -> [SupersetAPIError]\nclass SupersetClientBase:\n # [DEF:SupersetClientInit:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.\n # @RELATION: DEPENDS_ON -> [Environment]\n # @RELATION: DEPENDS_ON -> [APIClient]\n def __init__(self, env: Environment):\n with belief_scope(\"SupersetClientInit\"):\n app_logger.reason(\n \"Initializing Superset client for environment\",\n extra={\"environment\": getattr(env, \"id\", None), \"env_name\": env.name},\n )\n self.env = env\n # Construct auth payload expected by Superset API\n auth_payload = {\n \"username\": env.username,\n \"password\": env.password,\n \"provider\": \"db\",\n \"refresh\": \"true\",\n }\n self.network = APIClient(\n config={\"base_url\": env.url, \"auth\": auth_payload},\n verify_ssl=env.verify_ssl,\n timeout=env.timeout,\n )\n self.delete_before_reimport: bool = False\n app_logger.reflect(\n \"Superset client initialized\",\n extra={\"environment\": getattr(self.env, \"id\", None)},\n )\n\n # [/DEF:SupersetClientInit:Function]\n\n # [DEF:SupersetClientAuthenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticates the client using the configured credentials.\n # @RELATION: CALLS -> [APIClient]\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"SupersetClientAuthenticate\"):\n app_logger.reason(\n \"Authenticating Superset client\",\n extra={\"environment\": getattr(self.env, \"id\", None)},\n )\n tokens = self.network.authenticate()\n app_logger.reflect(\n \"Superset client authentication completed\",\n extra={\n \"environment\": getattr(self.env, \"id\", None),\n \"token_keys\": sorted(tokens.keys()),\n },\n )\n return tokens\n # [/DEF:SupersetClientAuthenticate:Function]\n\n @property\n # [DEF:SupersetClientHeaders:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.\n def headers(self) -> dict:\n with belief_scope(\"headers\"):\n return self.network.headers\n\n # [/DEF:SupersetClientHeaders:Function]\n\n # --- Pagination helpers ---\n\n # [DEF:SupersetClientValidateQueryParams:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Ensures query parameters have default page and page_size.\n def _validate_query_params(self, query: Optional[Dict]) -> Dict:\n with belief_scope(\"_validate_query_params\"):\n # Superset list endpoints commonly cap page_size at 100.\n # Using 100 avoids partial fetches when larger values are silently truncated.\n base_query = {\"page\": 0, \"page_size\": 100}\n return {**base_query, **(query or {})}\n\n # [/DEF:SupersetClientValidateQueryParams:Function]\n\n # [DEF:SupersetClientFetchTotalObjectCount:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Fetches the total number of items for a given endpoint.\n # @RELATION: CALLS -> [APIClient]\n def _fetch_total_object_count(self, endpoint: str) -> int:\n with belief_scope(\"_fetch_total_object_count\"):\n return self.network.fetch_paginated_count(\n endpoint=endpoint,\n query_params={\"page\": 0, \"page_size\": 1},\n count_field=\"count\",\n )\n\n # [/DEF:SupersetClientFetchTotalObjectCount:Function]\n\n # [DEF:SupersetClientFetchAllPages:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Iterates through all pages to collect all data items.\n # @RELATION: CALLS -> [APIClient]\n def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]:\n with belief_scope(\"_fetch_all_pages\"):\n return self.network.fetch_paginated_data(\n endpoint=endpoint, pagination_options=pagination_options\n )\n\n # [/DEF:SupersetClientFetchAllPages:Function]\n\n # --- Import/Export helpers ---\n\n # [DEF:SupersetClientDoImport:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Performs the actual multipart upload for import.\n # @RELATION: CALLS -> [APIClient]\n def _do_import(self, file_name: Union[str, Path]) -> Dict:\n with belief_scope(\"_do_import\"):\n app_logger.debug(f\"[_do_import][State] Uploading file: {file_name}\")\n file_path = Path(file_name)\n if not file_path.exists():\n app_logger.error(\n f\"[_do_import][Failure] File does not exist: {file_name}\"\n )\n raise FileNotFoundError(f\"File does not exist: {file_name}\")\n\n return self.network.upload_file(\n endpoint=\"/dashboard/import/\",\n file_info={\n \"file_obj\": file_path,\n \"file_name\": file_path.name,\n \"form_field\": \"formData\",\n },\n extra_data={\"overwrite\": \"true\"},\n timeout=self.env.timeout * 2,\n )\n\n # [/DEF:SupersetClientDoImport:Function]\n\n # [DEF:SupersetClientValidateExportResponse:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Validates that the export response is a non-empty ZIP archive.\n def _validate_export_response(self, response: Response, dashboard_id: int) -> None:\n with belief_scope(\"_validate_export_response\"):\n content_type = response.headers.get(\"Content-Type\", \"\")\n if \"application/zip\" not in content_type:\n raise SupersetAPIError(\n f\"Получен не ZIP-архив (Content-Type: {content_type})\"\n )\n if not response.content:\n raise SupersetAPIError(\"Получены пустые данные при экспорте\")\n\n # [/DEF:SupersetClientValidateExportResponse:Function]\n\n # [DEF:SupersetClientResolveExportFilename:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Determines the filename for an exported dashboard.\n def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:\n with belief_scope(\"_resolve_export_filename\"):\n filename = get_filename_from_headers(dict(response.headers))\n if not filename:\n from datetime import datetime\n\n timestamp = datetime.now().strftime(\"%Y%m%dT%H%M%S\")\n filename = f\"dashboard_export_{dashboard_id}_{timestamp}.zip\"\n app_logger.warning(\n \"[_resolve_export_filename][Warning] Generated filename: %s\",\n filename,\n )\n return filename\n\n # [/DEF:SupersetClientResolveExportFilename:Function]\n\n # [DEF:SupersetClientValidateImportFile:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.\n def _validate_import_file(self, zip_path: Union[str, Path]) -> None:\n with belief_scope(\"_validate_import_file\"):\n path = Path(zip_path)\n if not path.exists():\n raise FileNotFoundError(f\"Файл {zip_path} не существует\")\n if not zipfile.is_zipfile(path):\n raise SupersetAPIError(f\"Файл {zip_path} не является ZIP-архивом\")\n with zipfile.ZipFile(path, \"r\") as zf:\n if not any(n.endswith(\"metadata.yaml\") for n in zf.namelist()):\n raise SupersetAPIError(\n f\"Архив {zip_path} не содержит 'metadata.yaml'\"\n )\n\n # [/DEF:SupersetClientValidateImportFile:Function]\n\n # [DEF:SupersetClientResolveTargetIdForDelete:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Resolves a dashboard ID from either an ID or a slug.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def _resolve_target_id_for_delete(\n self, dash_id: Optional[int], dash_slug: Optional[str]\n ) -> Optional[int]:\n with belief_scope(\"_resolve_target_id_for_delete\"):\n if dash_id is not None:\n return dash_id\n if dash_slug is not None:\n app_logger.debug(\n \"[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.\",\n dash_slug,\n )\n try:\n _, candidates = self.get_dashboards(\n query={\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dash_slug}]\n }\n )\n if candidates:\n target_id = candidates[0][\"id\"]\n app_logger.debug(\n \"[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.\",\n target_id,\n )\n return target_id\n except Exception as e:\n app_logger.warning(\n \"[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s\",\n dash_slug,\n e,\n )\n return None\n\n # [/DEF:SupersetClientResolveTargetIdForDelete:Function]\n\n # [DEF:SupersetClientGetAllResources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_all_resources(\n self, resource_type: str, since_dttm: Optional[\"datetime\"] = None\n ) -> List[Dict]:\n with belief_scope(\n \"SupersetClient.get_all_resources\",\n f\"type={resource_type}, since={since_dttm}\",\n ):\n column_map = {\n \"chart\": {\n \"endpoint\": \"/chart/\",\n \"columns\": [\"id\", \"uuid\", \"slice_name\"],\n },\n \"dataset\": {\n \"endpoint\": \"/dataset/\",\n \"columns\": [\"id\", \"uuid\", \"table_name\"],\n },\n \"dashboard\": {\n \"endpoint\": \"/dashboard/\",\n \"columns\": [\"id\", \"uuid\", \"slug\", \"dashboard_title\"],\n },\n }\n config = column_map.get(resource_type)\n if not config:\n app_logger.warning(\n \"[get_all_resources][Warning] Unknown resource type: %s\",\n resource_type,\n )\n return []\n\n query = {\"columns\": config[\"columns\"]}\n\n if since_dttm:\n import math\n\n # Use int milliseconds to be safe\n timestamp_ms = math.floor(since_dttm.timestamp() * 1000)\n\n query[\"filters\"] = [\n {\"col\": \"changed_on_dttm\", \"opr\": \"gt\", \"value\": timestamp_ms}\n ]\n\n validated = self._validate_query_params(query)\n data = self._fetch_all_pages(\n endpoint=config[\"endpoint\"],\n pagination_options={\"base_query\": validated, \"results_field\": \"result\"},\n )\n app_logger.info(\n \"[get_all_resources][Exit] Fetched %d %s resources.\",\n len(data),\n resource_type,\n )\n return data\n\n # [/DEF:SupersetClientGetAllResources:Function]\n\n\n# [/DEF:SupersetClientBase:Class]\n# [/DEF:SupersetClientBase:Module]\n" + "body": "# [DEF:SupersetClientBase:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, api, client, base, pagination, auth, import, export\n# @PURPOSE: Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers.\n# @RELATION: DEPENDS_ON -> [ConfigModels]\n# @RELATION: DEPENDS_ON -> [APIClient]\n# @RELATION: DEPENDS_ON -> [SupersetAPIError]\n# @RELATION: DEPENDS_ON -> [get_filename_from_headers]\n\nimport json\nimport zipfile\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union, cast\nfrom requests import Response\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\nfrom ..utils.network import APIClient, SupersetAPIError\nfrom ..utils.fileio import get_filename_from_headers\nfrom ..config_models import Environment\n\nlog = MarkerLogger(\"SupersetClientBase\")\n\n\n# [DEF:SupersetClientBase:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Base class providing Superset client initialization, auth, pagination, and import/export plumbing.\n# @RELATION: DEPENDS_ON -> [ConfigModels]\n# @RELATION: DEPENDS_ON -> [APIClient]\n# @RELATION: DEPENDS_ON -> [SupersetAPIError]\nclass SupersetClientBase:\n # [DEF:SupersetClientInit:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.\n # @RELATION: DEPENDS_ON -> [Environment]\n # @RELATION: DEPENDS_ON -> [APIClient]\n def __init__(self, env: Environment):\n with belief_scope(\"SupersetClientInit\"):\n log.reason(\n \"Initializing Superset client for environment\",\n payload={\"environment\": getattr(env, \"id\", None), \"env_name\": env.name},\n )\n self.env = env\n # Construct auth payload expected by Superset API\n auth_payload = {\n \"username\": env.username,\n \"password\": env.password,\n \"provider\": \"db\",\n \"refresh\": \"true\",\n }\n self.network = APIClient(\n config={\"base_url\": env.url, \"auth\": auth_payload},\n verify_ssl=env.verify_ssl,\n timeout=env.timeout,\n )\n self.delete_before_reimport: bool = False\n log.reflect(\n \"Superset client initialized\",\n payload={\"environment\": getattr(self.env, \"id\", None)},\n )\n\n # [/DEF:SupersetClientInit:Function]\n\n # [DEF:SupersetClientAuthenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticates the client using the configured credentials.\n # @RELATION: CALLS -> [APIClient]\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"SupersetClientAuthenticate\"):\n log.reason(\n \"Authenticating Superset client\",\n payload={\"environment\": getattr(self.env, \"id\", None)},\n )\n tokens = self.network.authenticate()\n log.reflect(\n \"Superset client authentication completed\",\n payload={\n \"environment\": getattr(self.env, \"id\", None),\n \"token_keys\": sorted(tokens.keys()),\n },\n )\n return tokens\n # [/DEF:SupersetClientAuthenticate:Function]\n\n @property\n # [DEF:SupersetClientHeaders:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.\n def headers(self) -> dict:\n with belief_scope(\"headers\"):\n return self.network.headers\n\n # [/DEF:SupersetClientHeaders:Function]\n\n # --- Pagination helpers ---\n\n # [DEF:SupersetClientValidateQueryParams:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Ensures query parameters have default page and page_size.\n def _validate_query_params(self, query: Optional[Dict]) -> Dict:\n with belief_scope(\"_validate_query_params\"):\n # Superset list endpoints commonly cap page_size at 100.\n # Using 100 avoids partial fetches when larger values are silently truncated.\n base_query = {\"page\": 0, \"page_size\": 100}\n return {**base_query, **(query or {})}\n\n # [/DEF:SupersetClientValidateQueryParams:Function]\n\n # [DEF:SupersetClientFetchTotalObjectCount:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Fetches the total number of items for a given endpoint.\n # @RELATION: CALLS -> [APIClient]\n def _fetch_total_object_count(self, endpoint: str) -> int:\n with belief_scope(\"_fetch_total_object_count\"):\n return self.network.fetch_paginated_count(\n endpoint=endpoint,\n query_params={\"page\": 0, \"page_size\": 1},\n count_field=\"count\",\n )\n\n # [/DEF:SupersetClientFetchTotalObjectCount:Function]\n\n # [DEF:SupersetClientFetchAllPages:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Iterates through all pages to collect all data items.\n # @RELATION: CALLS -> [APIClient]\n def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]:\n with belief_scope(\"_fetch_all_pages\"):\n return self.network.fetch_paginated_data(\n endpoint=endpoint, pagination_options=pagination_options\n )\n\n # [/DEF:SupersetClientFetchAllPages:Function]\n\n # --- Import/Export helpers ---\n\n # [DEF:SupersetClientDoImport:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Performs the actual multipart upload for import.\n # @RELATION: CALLS -> [APIClient]\n def _do_import(self, file_name: Union[str, Path]) -> Dict:\n with belief_scope(\"_do_import\"):\n log.reason(\"Uploading file for import\", payload={\"file_name\": str(file_name)})\n file_path = Path(file_name)\n if not file_path.exists():\n log.explore(\"Import file does not exist\", error=\"FileNotFound\", payload={\"file_name\": str(file_name)})\n raise FileNotFoundError(f\"File does not exist: {file_name}\")\n\n return self.network.upload_file(\n endpoint=\"/dashboard/import/\",\n file_info={\n \"file_obj\": file_path,\n \"file_name\": file_path.name,\n \"form_field\": \"formData\",\n },\n extra_data={\"overwrite\": \"true\"},\n timeout=self.env.timeout * 2,\n )\n\n # [/DEF:SupersetClientDoImport:Function]\n\n # [DEF:SupersetClientValidateExportResponse:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Validates that the export response is a non-empty ZIP archive.\n def _validate_export_response(self, response: Response, dashboard_id: int) -> None:\n with belief_scope(\"_validate_export_response\"):\n content_type = response.headers.get(\"Content-Type\", \"\")\n if \"application/zip\" not in content_type:\n raise SupersetAPIError(\n f\"Получен не ZIP-архив (Content-Type: {content_type})\"\n )\n if not response.content:\n raise SupersetAPIError(\"Получены пустые данные при экспорте\")\n\n # [/DEF:SupersetClientValidateExportResponse:Function]\n\n # [DEF:SupersetClientResolveExportFilename:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Determines the filename for an exported dashboard.\n def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:\n with belief_scope(\"_resolve_export_filename\"):\n filename = get_filename_from_headers(dict(response.headers))\n if not filename:\n from datetime import datetime\n\n timestamp = datetime.now().strftime(\"%Y%m%dT%H%M%S\")\n filename = f\"dashboard_export_{dashboard_id}_{timestamp}.zip\"\n log.reflect(\n \"Export filename not found in headers, using generated name\",\n payload={\"dashboard_id\": dashboard_id, \"generated_filename\": filename},\n )\n return filename\n\n # [/DEF:SupersetClientResolveExportFilename:Function]\n\n # [DEF:SupersetClientValidateImportFile:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.\n def _validate_import_file(self, zip_path: Union[str, Path]) -> None:\n with belief_scope(\"_validate_import_file\"):\n path = Path(zip_path)\n if not path.exists():\n raise FileNotFoundError(f\"Файл {zip_path} не существует\")\n if not zipfile.is_zipfile(path):\n raise SupersetAPIError(f\"Файл {zip_path} не является ZIP-архивом\")\n with zipfile.ZipFile(path, \"r\") as zf:\n if not any(n.endswith(\"metadata.yaml\") for n in zf.namelist()):\n raise SupersetAPIError(\n f\"Архив {zip_path} не содержит 'metadata.yaml'\"\n )\n\n # [/DEF:SupersetClientValidateImportFile:Function]\n\n # [DEF:SupersetClientResolveTargetIdForDelete:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Resolves a dashboard ID from either an ID or a slug.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def _resolve_target_id_for_delete(\n self, dash_id: Optional[int], dash_slug: Optional[str]\n ) -> Optional[int]:\n with belief_scope(\"_resolve_target_id_for_delete\"):\n if dash_id is not None:\n return dash_id\n if dash_slug is not None:\n log.reason(\"Resolving dashboard ID by slug\", payload={\"slug\": dash_slug})\n try:\n _, candidates = self.get_dashboards(\n query={\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dash_slug}]\n }\n )\n if candidates:\n target_id = candidates[0][\"id\"]\n log.reflect(\"Resolved slug to dashboard ID\", payload={\"slug\": dash_slug, \"target_id\": target_id})\n return target_id\n except Exception as e:\n log.explore(\"Could not resolve slug to dashboard ID\", error=str(e), payload={\"slug\": dash_slug})\n return None\n\n # [/DEF:SupersetClientResolveTargetIdForDelete:Function]\n\n # [DEF:SupersetClientGetAllResources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_all_resources(\n self, resource_type: str, since_dttm: Optional[\"datetime\"] = None\n ) -> List[Dict]:\n with belief_scope(\n \"SupersetClient.get_all_resources\",\n f\"type={resource_type}, since={since_dttm}\",\n ):\n column_map = {\n \"chart\": {\n \"endpoint\": \"/chart/\",\n \"columns\": [\"id\", \"uuid\", \"slice_name\"],\n },\n \"dataset\": {\n \"endpoint\": \"/dataset/\",\n \"columns\": [\"id\", \"uuid\", \"table_name\"],\n },\n \"dashboard\": {\n \"endpoint\": \"/dashboard/\",\n \"columns\": [\"id\", \"uuid\", \"slug\", \"dashboard_title\"],\n },\n }\n config = column_map.get(resource_type)\n if not config:\n log.explore(\"Unknown resource type requested\", error=f\"Invalid resource type: {resource_type}\", payload={\"resource_type\": resource_type})\n return []\n\n query = {\"columns\": config[\"columns\"]}\n\n if since_dttm:\n import math\n\n # Use int milliseconds to be safe\n timestamp_ms = math.floor(since_dttm.timestamp() * 1000)\n\n query[\"filters\"] = [\n {\"col\": \"changed_on_dttm\", \"opr\": \"gt\", \"value\": timestamp_ms}\n ]\n\n validated = self._validate_query_params(query)\n data = self._fetch_all_pages(\n endpoint=config[\"endpoint\"],\n pagination_options={\"base_query\": validated, \"results_field\": \"result\"},\n )\n log.reflect(\n \"Resources fetched\",\n payload={\"resource_type\": resource_type, \"count\": len(data)},\n )\n return data\n\n # [/DEF:SupersetClientGetAllResources:Function]\n\n\n# [/DEF:SupersetClientBase:Class]\n# [/DEF:SupersetClientBase:Module]\n" }, { "contract_id": "SupersetClientInit", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 32, - "end_line": 62, + "start_line": 33, + "end_line": 63, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39114,14 +39870,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientInit:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.\n # @RELATION: DEPENDS_ON -> [Environment]\n # @RELATION: DEPENDS_ON -> [APIClient]\n def __init__(self, env: Environment):\n with belief_scope(\"SupersetClientInit\"):\n app_logger.reason(\n \"Initializing Superset client for environment\",\n extra={\"environment\": getattr(env, \"id\", None), \"env_name\": env.name},\n )\n self.env = env\n # Construct auth payload expected by Superset API\n auth_payload = {\n \"username\": env.username,\n \"password\": env.password,\n \"provider\": \"db\",\n \"refresh\": \"true\",\n }\n self.network = APIClient(\n config={\"base_url\": env.url, \"auth\": auth_payload},\n verify_ssl=env.verify_ssl,\n timeout=env.timeout,\n )\n self.delete_before_reimport: bool = False\n app_logger.reflect(\n \"Superset client initialized\",\n extra={\"environment\": getattr(self.env, \"id\", None)},\n )\n\n # [/DEF:SupersetClientInit:Function]\n" + "body": " # [DEF:SupersetClientInit:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.\n # @RELATION: DEPENDS_ON -> [Environment]\n # @RELATION: DEPENDS_ON -> [APIClient]\n def __init__(self, env: Environment):\n with belief_scope(\"SupersetClientInit\"):\n log.reason(\n \"Initializing Superset client for environment\",\n payload={\"environment\": getattr(env, \"id\", None), \"env_name\": env.name},\n )\n self.env = env\n # Construct auth payload expected by Superset API\n auth_payload = {\n \"username\": env.username,\n \"password\": env.password,\n \"provider\": \"db\",\n \"refresh\": \"true\",\n }\n self.network = APIClient(\n config={\"base_url\": env.url, \"auth\": auth_payload},\n verify_ssl=env.verify_ssl,\n timeout=env.timeout,\n )\n self.delete_before_reimport: bool = False\n log.reflect(\n \"Superset client initialized\",\n payload={\"environment\": getattr(self.env, \"id\", None)},\n )\n\n # [/DEF:SupersetClientInit:Function]\n" }, { "contract_id": "SupersetClientAuthenticate", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 64, - "end_line": 83, + "start_line": 65, + "end_line": 84, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39138,14 +39894,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientAuthenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticates the client using the configured credentials.\n # @RELATION: CALLS -> [APIClient]\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"SupersetClientAuthenticate\"):\n app_logger.reason(\n \"Authenticating Superset client\",\n extra={\"environment\": getattr(self.env, \"id\", None)},\n )\n tokens = self.network.authenticate()\n app_logger.reflect(\n \"Superset client authentication completed\",\n extra={\n \"environment\": getattr(self.env, \"id\", None),\n \"token_keys\": sorted(tokens.keys()),\n },\n )\n return tokens\n # [/DEF:SupersetClientAuthenticate:Function]\n" + "body": " # [DEF:SupersetClientAuthenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticates the client using the configured credentials.\n # @RELATION: CALLS -> [APIClient]\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"SupersetClientAuthenticate\"):\n log.reason(\n \"Authenticating Superset client\",\n payload={\"environment\": getattr(self.env, \"id\", None)},\n )\n tokens = self.network.authenticate()\n log.reflect(\n \"Superset client authentication completed\",\n payload={\n \"environment\": getattr(self.env, \"id\", None),\n \"token_keys\": sorted(tokens.keys()),\n },\n )\n return tokens\n # [/DEF:SupersetClientAuthenticate:Function]\n" }, { "contract_id": "SupersetClientHeaders", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 86, - "end_line": 93, + "start_line": 87, + "end_line": 94, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39161,8 +39917,8 @@ "contract_id": "SupersetClientValidateQueryParams", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 97, - "end_line": 107, + "start_line": 98, + "end_line": 108, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39178,8 +39934,8 @@ "contract_id": "SupersetClientFetchTotalObjectCount", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 109, - "end_line": 121, + "start_line": 110, + "end_line": 122, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39212,8 +39968,8 @@ "contract_id": "SupersetClientFetchAllPages", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 123, - "end_line": 133, + "start_line": 124, + "end_line": 134, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39246,8 +40002,8 @@ "contract_id": "SupersetClientDoImport", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 137, - "end_line": 162, + "start_line": 138, + "end_line": 161, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39274,14 +40030,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientDoImport:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Performs the actual multipart upload for import.\n # @RELATION: CALLS -> [APIClient]\n def _do_import(self, file_name: Union[str, Path]) -> Dict:\n with belief_scope(\"_do_import\"):\n app_logger.debug(f\"[_do_import][State] Uploading file: {file_name}\")\n file_path = Path(file_name)\n if not file_path.exists():\n app_logger.error(\n f\"[_do_import][Failure] File does not exist: {file_name}\"\n )\n raise FileNotFoundError(f\"File does not exist: {file_name}\")\n\n return self.network.upload_file(\n endpoint=\"/dashboard/import/\",\n file_info={\n \"file_obj\": file_path,\n \"file_name\": file_path.name,\n \"form_field\": \"formData\",\n },\n extra_data={\"overwrite\": \"true\"},\n timeout=self.env.timeout * 2,\n )\n\n # [/DEF:SupersetClientDoImport:Function]\n" + "body": " # [DEF:SupersetClientDoImport:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Performs the actual multipart upload for import.\n # @RELATION: CALLS -> [APIClient]\n def _do_import(self, file_name: Union[str, Path]) -> Dict:\n with belief_scope(\"_do_import\"):\n log.reason(\"Uploading file for import\", payload={\"file_name\": str(file_name)})\n file_path = Path(file_name)\n if not file_path.exists():\n log.explore(\"Import file does not exist\", error=\"FileNotFound\", payload={\"file_name\": str(file_name)})\n raise FileNotFoundError(f\"File does not exist: {file_name}\")\n\n return self.network.upload_file(\n endpoint=\"/dashboard/import/\",\n file_info={\n \"file_obj\": file_path,\n \"file_name\": file_path.name,\n \"form_field\": \"formData\",\n },\n extra_data={\"overwrite\": \"true\"},\n timeout=self.env.timeout * 2,\n )\n\n # [/DEF:SupersetClientDoImport:Function]\n" }, { "contract_id": "SupersetClientValidateExportResponse", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 164, - "end_line": 177, + "start_line": 163, + "end_line": 176, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39297,8 +40053,8 @@ "contract_id": "SupersetClientResolveExportFilename", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 179, - "end_line": 196, + "start_line": 178, + "end_line": 195, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39308,14 +40064,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientResolveExportFilename:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Determines the filename for an exported dashboard.\n def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:\n with belief_scope(\"_resolve_export_filename\"):\n filename = get_filename_from_headers(dict(response.headers))\n if not filename:\n from datetime import datetime\n\n timestamp = datetime.now().strftime(\"%Y%m%dT%H%M%S\")\n filename = f\"dashboard_export_{dashboard_id}_{timestamp}.zip\"\n app_logger.warning(\n \"[_resolve_export_filename][Warning] Generated filename: %s\",\n filename,\n )\n return filename\n\n # [/DEF:SupersetClientResolveExportFilename:Function]\n" + "body": " # [DEF:SupersetClientResolveExportFilename:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Determines the filename for an exported dashboard.\n def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:\n with belief_scope(\"_resolve_export_filename\"):\n filename = get_filename_from_headers(dict(response.headers))\n if not filename:\n from datetime import datetime\n\n timestamp = datetime.now().strftime(\"%Y%m%dT%H%M%S\")\n filename = f\"dashboard_export_{dashboard_id}_{timestamp}.zip\"\n log.reflect(\n \"Export filename not found in headers, using generated name\",\n payload={\"dashboard_id\": dashboard_id, \"generated_filename\": filename},\n )\n return filename\n\n # [/DEF:SupersetClientResolveExportFilename:Function]\n" }, { "contract_id": "SupersetClientValidateImportFile", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 198, - "end_line": 214, + "start_line": 197, + "end_line": 213, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39331,8 +40087,8 @@ "contract_id": "SupersetClientResolveTargetIdForDelete", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 216, - "end_line": 252, + "start_line": 215, + "end_line": 241, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -39359,14 +40115,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientResolveTargetIdForDelete:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Resolves a dashboard ID from either an ID or a slug.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def _resolve_target_id_for_delete(\n self, dash_id: Optional[int], dash_slug: Optional[str]\n ) -> Optional[int]:\n with belief_scope(\"_resolve_target_id_for_delete\"):\n if dash_id is not None:\n return dash_id\n if dash_slug is not None:\n app_logger.debug(\n \"[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.\",\n dash_slug,\n )\n try:\n _, candidates = self.get_dashboards(\n query={\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dash_slug}]\n }\n )\n if candidates:\n target_id = candidates[0][\"id\"]\n app_logger.debug(\n \"[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.\",\n target_id,\n )\n return target_id\n except Exception as e:\n app_logger.warning(\n \"[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s\",\n dash_slug,\n e,\n )\n return None\n\n # [/DEF:SupersetClientResolveTargetIdForDelete:Function]\n" + "body": " # [DEF:SupersetClientResolveTargetIdForDelete:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Resolves a dashboard ID from either an ID or a slug.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def _resolve_target_id_for_delete(\n self, dash_id: Optional[int], dash_slug: Optional[str]\n ) -> Optional[int]:\n with belief_scope(\"_resolve_target_id_for_delete\"):\n if dash_id is not None:\n return dash_id\n if dash_slug is not None:\n log.reason(\"Resolving dashboard ID by slug\", payload={\"slug\": dash_slug})\n try:\n _, candidates = self.get_dashboards(\n query={\n \"filters\": [{\"col\": \"slug\", \"op\": \"eq\", \"value\": dash_slug}]\n }\n )\n if candidates:\n target_id = candidates[0][\"id\"]\n log.reflect(\"Resolved slug to dashboard ID\", payload={\"slug\": dash_slug, \"target_id\": target_id})\n return target_id\n except Exception as e:\n log.explore(\"Could not resolve slug to dashboard ID\", error=str(e), payload={\"slug\": dash_slug})\n return None\n\n # [/DEF:SupersetClientResolveTargetIdForDelete:Function]\n" }, { "contract_id": "SupersetClientGetAllResources", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_base.py", - "start_line": 254, - "end_line": 311, + "start_line": 243, + "end_line": 296, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39383,7 +40139,7 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetAllResources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_all_resources(\n self, resource_type: str, since_dttm: Optional[\"datetime\"] = None\n ) -> List[Dict]:\n with belief_scope(\n \"SupersetClient.get_all_resources\",\n f\"type={resource_type}, since={since_dttm}\",\n ):\n column_map = {\n \"chart\": {\n \"endpoint\": \"/chart/\",\n \"columns\": [\"id\", \"uuid\", \"slice_name\"],\n },\n \"dataset\": {\n \"endpoint\": \"/dataset/\",\n \"columns\": [\"id\", \"uuid\", \"table_name\"],\n },\n \"dashboard\": {\n \"endpoint\": \"/dashboard/\",\n \"columns\": [\"id\", \"uuid\", \"slug\", \"dashboard_title\"],\n },\n }\n config = column_map.get(resource_type)\n if not config:\n app_logger.warning(\n \"[get_all_resources][Warning] Unknown resource type: %s\",\n resource_type,\n )\n return []\n\n query = {\"columns\": config[\"columns\"]}\n\n if since_dttm:\n import math\n\n # Use int milliseconds to be safe\n timestamp_ms = math.floor(since_dttm.timestamp() * 1000)\n\n query[\"filters\"] = [\n {\"col\": \"changed_on_dttm\", \"opr\": \"gt\", \"value\": timestamp_ms}\n ]\n\n validated = self._validate_query_params(query)\n data = self._fetch_all_pages(\n endpoint=config[\"endpoint\"],\n pagination_options={\"base_query\": validated, \"results_field\": \"result\"},\n )\n app_logger.info(\n \"[get_all_resources][Exit] Fetched %d %s resources.\",\n len(data),\n resource_type,\n )\n return data\n\n # [/DEF:SupersetClientGetAllResources:Function]\n" + "body": " # [DEF:SupersetClientGetAllResources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_all_resources(\n self, resource_type: str, since_dttm: Optional[\"datetime\"] = None\n ) -> List[Dict]:\n with belief_scope(\n \"SupersetClient.get_all_resources\",\n f\"type={resource_type}, since={since_dttm}\",\n ):\n column_map = {\n \"chart\": {\n \"endpoint\": \"/chart/\",\n \"columns\": [\"id\", \"uuid\", \"slice_name\"],\n },\n \"dataset\": {\n \"endpoint\": \"/dataset/\",\n \"columns\": [\"id\", \"uuid\", \"table_name\"],\n },\n \"dashboard\": {\n \"endpoint\": \"/dashboard/\",\n \"columns\": [\"id\", \"uuid\", \"slug\", \"dashboard_title\"],\n },\n }\n config = column_map.get(resource_type)\n if not config:\n log.explore(\"Unknown resource type requested\", error=f\"Invalid resource type: {resource_type}\", payload={\"resource_type\": resource_type})\n return []\n\n query = {\"columns\": config[\"columns\"]}\n\n if since_dttm:\n import math\n\n # Use int milliseconds to be safe\n timestamp_ms = math.floor(since_dttm.timestamp() * 1000)\n\n query[\"filters\"] = [\n {\"col\": \"changed_on_dttm\", \"opr\": \"gt\", \"value\": timestamp_ms}\n ]\n\n validated = self._validate_query_params(query)\n data = self._fetch_all_pages(\n endpoint=config[\"endpoint\"],\n pagination_options={\"base_query\": validated, \"results_field\": \"result\"},\n )\n log.reflect(\n \"Resources fetched\",\n payload={\"resource_type\": resource_type, \"count\": len(data)},\n )\n return data\n\n # [/DEF:SupersetClientGetAllResources:Function]\n" }, { "contract_id": "SupersetChartsMixin", @@ -39487,7 +40243,7 @@ "contract_type": "Module", "file_path": "backend/src/core/superset_client/_dashboards_crud.py", "start_line": 1, - "end_line": 376, + "end_line": 353, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39526,14 +40282,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetDashboardsCrudMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, dashboards, crud, detail, export, import, delete\n# @PURPOSE: Dashboard CRUD mixin for SupersetClient — detail, export, import, delete.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]\n# @RELATION: DEPENDS_ON -> [SupersetChartsMixin]\n\nimport json\nimport re\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union, cast\nfrom requests import Response\n\nfrom ..logger import logger as app_logger, belief_scope\n\napp_logger = cast(Any, app_logger)\n\n\n# [DEF:SupersetDashboardsCrudMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing dashboard detail resolution, export, import, and delete operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]\n# @RELATION: DEPENDS_ON -> [SupersetChartsMixin]\nclass SupersetDashboardsCrudMixin:\n # [DEF:SupersetClientGetDashboardDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dashboard information including related charts and datasets.\n # @RELATION: CALLS -> [SupersetClientGetDashboard]\n # @RELATION: CALLS -> [SupersetClientGetChart]\n def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:\n with belief_scope(\n \"SupersetClient.get_dashboard_detail\", f\"ref={dashboard_ref}\"\n ):\n dashboard_response = self.get_dashboard(dashboard_ref)\n dashboard_data = dashboard_response.get(\"result\", dashboard_response)\n\n charts: List[Dict] = []\n datasets: List[Dict] = []\n\n # [DEF:extract_dataset_id_from_form_data:Function]\n def extract_dataset_id_from_form_data(\n form_data: Optional[Dict],\n ) -> Optional[int]:\n if not isinstance(form_data, dict):\n return None\n datasource = form_data.get(\"datasource\")\n if isinstance(datasource, str):\n matched = re.match(r\"^(\\d+)__\", datasource)\n if matched:\n try:\n return int(matched.group(1))\n except ValueError:\n return None\n if isinstance(datasource, dict):\n ds_id = datasource.get(\"id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n ds_id = form_data.get(\"datasource_id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n\n # [/DEF:extract_dataset_id_from_form_data:Function]\n\n try:\n charts_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/charts\"\n )\n charts_payload = (\n charts_response.get(\"result\", [])\n if isinstance(charts_response, dict)\n else []\n )\n for chart_obj in charts_payload:\n if not isinstance(chart_obj, dict):\n continue\n chart_id = chart_obj.get(\"id\")\n if chart_id is None:\n continue\n form_data = chart_obj.get(\"form_data\")\n if isinstance(form_data, str):\n try:\n form_data = json.loads(form_data)\n except Exception:\n form_data = {}\n dataset_id = extract_dataset_id_from_form_data(\n form_data\n ) or chart_obj.get(\"datasource_id\")\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_obj.get(\"slice_name\")\n or chart_obj.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": (\n form_data.get(\"viz_type\")\n if isinstance(form_data, dict) else None\n ),\n \"dataset_id\": int(dataset_id) if dataset_id is not None else None,\n \"last_modified\": chart_obj.get(\"changed_on\"),\n \"overview\": chart_obj.get(\"description\")\n or (form_data.get(\"viz_type\") if isinstance(form_data, dict) else None)\n or \"Chart\",\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s\", e,\n )\n\n try:\n datasets_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/datasets\"\n )\n datasets_payload = (\n datasets_response.get(\"result\", [])\n if isinstance(datasets_response, dict)\n else []\n )\n for dataset_obj in datasets_payload:\n if not isinstance(dataset_obj, dict):\n continue\n dataset_id = dataset_obj.get(\"id\")\n if dataset_id is None:\n continue\n db_payload = dataset_obj.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_obj.get(\"table_name\")\n or dataset_obj.get(\"datasource_name\")\n or dataset_obj.get(\"name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_obj.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or dataset_obj.get(\"database_name\") or \"Unknown\",\n \"last_modified\": dataset_obj.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s\", e,\n )\n\n # Fallback: derive chart IDs from layout metadata\n if not charts:\n raw_position_json = dashboard_data.get(\"position_json\")\n chart_ids_from_position = set()\n if isinstance(raw_position_json, str) and raw_position_json:\n try:\n parsed_position = json.loads(raw_position_json)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_position)\n )\n except Exception:\n pass\n elif isinstance(raw_position_json, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_position_json)\n )\n\n raw_json_metadata = dashboard_data.get(\"json_metadata\")\n if isinstance(raw_json_metadata, str) and raw_json_metadata:\n try:\n parsed_metadata = json.loads(raw_json_metadata)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_metadata)\n )\n except Exception:\n pass\n elif isinstance(raw_json_metadata, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_json_metadata)\n )\n\n app_logger.info(\n \"[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)\",\n len(chart_ids_from_position), dashboard_ref,\n )\n\n for chart_id in sorted(chart_ids_from_position):\n try:\n chart_response = self.get_chart(int(chart_id))\n chart_data = chart_response.get(\"result\", chart_response)\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_data.get(\"slice_name\")\n or chart_data.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": chart_data.get(\"viz_type\"),\n \"dataset_id\": chart_data.get(\"datasource_id\"),\n \"last_modified\": chart_data.get(\"changed_on\"),\n \"overview\": chart_data.get(\"description\")\n or chart_data.get(\"viz_type\") or \"Chart\",\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s\",\n chart_id, e,\n )\n\n # Backfill datasets from chart datasource IDs.\n dataset_ids_from_charts = {\n c.get(\"dataset_id\") for c in charts if c.get(\"dataset_id\") is not None\n }\n known_dataset_ids = {\n d.get(\"id\") for d in datasets if d.get(\"id\") is not None\n }\n missing_dataset_ids: List[int] = []\n for raw_dataset_id in dataset_ids_from_charts:\n if raw_dataset_id is None or raw_dataset_id in known_dataset_ids:\n continue\n try:\n missing_dataset_ids.append(int(raw_dataset_id))\n except (TypeError, ValueError):\n continue\n\n for dataset_id in missing_dataset_ids:\n try:\n dataset_response = self.get_dataset(int(dataset_id))\n dataset_data = dataset_response.get(\"result\", dataset_response)\n db_payload = dataset_data.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_data.get(\"table_name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_data.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or \"Unknown\",\n \"last_modified\": dataset_data.get(\"changed_on_utc\")\n or dataset_data.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s\",\n dataset_id, e,\n )\n\n unique_charts = {chart[\"id\"]: chart for chart in charts}\n unique_datasets = {dataset[\"id\"]: dataset for dataset in datasets}\n\n resolved_dashboard_id = dashboard_data.get(\"id\", dashboard_ref)\n return {\n \"id\": resolved_dashboard_id,\n \"title\": dashboard_data.get(\"dashboard_title\")\n or dashboard_data.get(\"title\") or f\"Dashboard {resolved_dashboard_id}\",\n \"slug\": dashboard_data.get(\"slug\"),\n \"url\": dashboard_data.get(\"url\"),\n \"description\": dashboard_data.get(\"description\") or \"\",\n \"last_modified\": dashboard_data.get(\"changed_on_utc\")\n or dashboard_data.get(\"changed_on\"),\n \"published\": dashboard_data.get(\"published\"),\n \"charts\": list(unique_charts.values()),\n \"datasets\": list(unique_datasets.values()),\n \"chart_count\": len(unique_charts),\n \"dataset_count\": len(unique_datasets),\n }\n\n # [/DEF:SupersetClientGetDashboardDetail:Function]\n\n # [DEF:SupersetClientExportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Экспортирует дашборд в виде ZIP-архива.\n # @SIDE_EFFECT: Performs network I/O to download archive.\n # @RELATION: CALLS -> [APIClient]\n def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:\n with belief_scope(\"export_dashboard\"):\n app_logger.info(\n \"[export_dashboard][Enter] Exporting dashboard %s.\", dashboard_id\n )\n response = self.network.request(\n method=\"GET\",\n endpoint=\"/dashboard/export/\",\n params={\"q\": json.dumps([dashboard_id])},\n stream=True,\n raw_response=True,\n )\n response = cast(Response, response)\n self._validate_export_response(response, dashboard_id)\n filename = self._resolve_export_filename(response, dashboard_id)\n app_logger.info(\n \"[export_dashboard][Exit] Exported dashboard %s to %s.\",\n dashboard_id, filename,\n )\n return response.content, filename\n\n # [/DEF:SupersetClientExportDashboard:Function]\n\n # [DEF:SupersetClientImportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Импортирует дашборд из ZIP-файла.\n # @SIDE_EFFECT: Performs network I/O to upload archive.\n # @RELATION: CALLS -> [SupersetClientDoImport]\n # @RELATION: CALLS -> [APIClient]\n def import_dashboard(\n self,\n file_name: Union[str, Path],\n dash_id: Optional[int] = None,\n dash_slug: Optional[str] = None,\n ) -> Dict:\n with belief_scope(\"import_dashboard\"):\n if file_name is None:\n raise ValueError(\"file_name cannot be None\")\n file_path = str(file_name)\n self._validate_import_file(file_path)\n try:\n return self._do_import(file_path)\n except Exception as exc:\n app_logger.error(\n \"[import_dashboard][Failure] First import attempt failed: %s\",\n exc, exc_info=True,\n )\n if not self.delete_before_reimport:\n raise\n\n target_id = self._resolve_target_id_for_delete(dash_id, dash_slug)\n if target_id is None:\n app_logger.error(\n \"[import_dashboard][Failure] No ID available for delete-retry.\"\n )\n raise\n\n self.delete_dashboard(target_id)\n app_logger.info(\n \"[import_dashboard][State] Deleted dashboard ID %s, retrying import.\",\n target_id,\n )\n return self._do_import(file_path)\n\n # [/DEF:SupersetClientImportDashboard:Function]\n\n # [DEF:SupersetClientDeleteDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Удаляет дашборд по его ID или slug.\n # @SIDE_EFFECT: Deletes resource from upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:\n with belief_scope(\"delete_dashboard\"):\n app_logger.info(\n \"[delete_dashboard][Enter] Deleting dashboard %s.\", dashboard_id\n )\n response = self.network.request(\n method=\"DELETE\", endpoint=f\"/dashboard/{dashboard_id}\"\n )\n response = cast(Dict, response)\n if response.get(\"result\", True) is not False:\n app_logger.info(\n \"[delete_dashboard][Success] Dashboard %s deleted.\", dashboard_id\n )\n else:\n app_logger.warning(\n \"[delete_dashboard][Warning] Unexpected response while deleting %s: %s\",\n dashboard_id, response,\n )\n\n # [/DEF:SupersetClientDeleteDashboard:Function]\n\n\n# [/DEF:SupersetDashboardsCrudMixin:Class]\n# [/DEF:SupersetDashboardsCrudMixin:Module]\n" + "body": "# [DEF:SupersetDashboardsCrudMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, dashboards, crud, detail, export, import, delete\n# @PURPOSE: Dashboard CRUD mixin for SupersetClient — detail, export, import, delete.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]\n# @RELATION: DEPENDS_ON -> [SupersetChartsMixin]\n\nimport json\nimport re\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union, cast\nfrom requests import Response\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\n\nlog = MarkerLogger(\"SupersetDashboardsCrud\")\n\n\n# [DEF:SupersetDashboardsCrudMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing dashboard detail resolution, export, import, and delete operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDashboardsFiltersMixin]\n# @RELATION: DEPENDS_ON -> [SupersetChartsMixin]\nclass SupersetDashboardsCrudMixin:\n # [DEF:SupersetClientGetDashboardDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dashboard information including related charts and datasets.\n # @RELATION: CALLS -> [SupersetClientGetDashboard]\n # @RELATION: CALLS -> [SupersetClientGetChart]\n def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:\n with belief_scope(\n \"SupersetClient.get_dashboard_detail\", f\"ref={dashboard_ref}\"\n ):\n dashboard_response = self.get_dashboard(dashboard_ref)\n dashboard_data = dashboard_response.get(\"result\", dashboard_response)\n\n charts: List[Dict] = []\n datasets: List[Dict] = []\n\n # [DEF:extract_dataset_id_from_form_data:Function]\n def extract_dataset_id_from_form_data(\n form_data: Optional[Dict],\n ) -> Optional[int]:\n if not isinstance(form_data, dict):\n return None\n datasource = form_data.get(\"datasource\")\n if isinstance(datasource, str):\n matched = re.match(r\"^(\\d+)__\", datasource)\n if matched:\n try:\n return int(matched.group(1))\n except ValueError:\n return None\n if isinstance(datasource, dict):\n ds_id = datasource.get(\"id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n ds_id = form_data.get(\"datasource_id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n\n # [/DEF:extract_dataset_id_from_form_data:Function]\n\n try:\n charts_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/charts\"\n )\n charts_payload = (\n charts_response.get(\"result\", [])\n if isinstance(charts_response, dict)\n else []\n )\n for chart_obj in charts_payload:\n if not isinstance(chart_obj, dict):\n continue\n chart_id = chart_obj.get(\"id\")\n if chart_id is None:\n continue\n form_data = chart_obj.get(\"form_data\")\n if isinstance(form_data, str):\n try:\n form_data = json.loads(form_data)\n except Exception:\n form_data = {}\n dataset_id = extract_dataset_id_from_form_data(\n form_data\n ) or chart_obj.get(\"datasource_id\")\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_obj.get(\"slice_name\")\n or chart_obj.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": (\n form_data.get(\"viz_type\")\n if isinstance(form_data, dict) else None\n ),\n \"dataset_id\": int(dataset_id) if dataset_id is not None else None,\n \"last_modified\": chart_obj.get(\"changed_on\"),\n \"overview\": chart_obj.get(\"description\")\n or (form_data.get(\"viz_type\") if isinstance(form_data, dict) else None)\n or \"Chart\",\n })\n except Exception as e:\n log.explore(\"Failed to fetch dashboard charts\", error=str(e), payload={\"dashboard_ref\": dashboard_ref})\n\n try:\n datasets_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/datasets\"\n )\n datasets_payload = (\n datasets_response.get(\"result\", [])\n if isinstance(datasets_response, dict)\n else []\n )\n for dataset_obj in datasets_payload:\n if not isinstance(dataset_obj, dict):\n continue\n dataset_id = dataset_obj.get(\"id\")\n if dataset_id is None:\n continue\n db_payload = dataset_obj.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_obj.get(\"table_name\")\n or dataset_obj.get(\"datasource_name\")\n or dataset_obj.get(\"name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_obj.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or dataset_obj.get(\"database_name\") or \"Unknown\",\n \"last_modified\": dataset_obj.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n log.explore(\"Failed to fetch dashboard datasets\", error=str(e), payload={\"dashboard_ref\": dashboard_ref})\n\n # Fallback: derive chart IDs from layout metadata\n if not charts:\n raw_position_json = dashboard_data.get(\"position_json\")\n chart_ids_from_position = set()\n if isinstance(raw_position_json, str) and raw_position_json:\n try:\n parsed_position = json.loads(raw_position_json)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_position)\n )\n except Exception:\n pass\n elif isinstance(raw_position_json, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_position_json)\n )\n\n raw_json_metadata = dashboard_data.get(\"json_metadata\")\n if isinstance(raw_json_metadata, str) and raw_json_metadata:\n try:\n parsed_metadata = json.loads(raw_json_metadata)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_metadata)\n )\n except Exception:\n pass\n elif isinstance(raw_json_metadata, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_json_metadata)\n )\n\n log.reason(\n \"Extracted fallback chart IDs from layout\",\n payload={\n \"chart_count\": len(chart_ids_from_position),\n \"dashboard_ref\": dashboard_ref,\n },\n )\n\n for chart_id in sorted(chart_ids_from_position):\n try:\n chart_response = self.get_chart(int(chart_id))\n chart_data = chart_response.get(\"result\", chart_response)\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_data.get(\"slice_name\")\n or chart_data.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": chart_data.get(\"viz_type\"),\n \"dataset_id\": chart_data.get(\"datasource_id\"),\n \"last_modified\": chart_data.get(\"changed_on\"),\n \"overview\": chart_data.get(\"description\")\n or chart_data.get(\"viz_type\") or \"Chart\",\n })\n except Exception as e:\n log.explore(\"Failed to resolve fallback chart\", error=str(e), payload={\"chart_id\": chart_id})\n\n # Backfill datasets from chart datasource IDs.\n dataset_ids_from_charts = {\n c.get(\"dataset_id\") for c in charts if c.get(\"dataset_id\") is not None\n }\n known_dataset_ids = {\n d.get(\"id\") for d in datasets if d.get(\"id\") is not None\n }\n missing_dataset_ids: List[int] = []\n for raw_dataset_id in dataset_ids_from_charts:\n if raw_dataset_id is None or raw_dataset_id in known_dataset_ids:\n continue\n try:\n missing_dataset_ids.append(int(raw_dataset_id))\n except (TypeError, ValueError):\n continue\n\n for dataset_id in missing_dataset_ids:\n try:\n dataset_response = self.get_dataset(int(dataset_id))\n dataset_data = dataset_response.get(\"result\", dataset_response)\n db_payload = dataset_data.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_data.get(\"table_name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_data.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or \"Unknown\",\n \"last_modified\": dataset_data.get(\"changed_on_utc\")\n or dataset_data.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n log.explore(\"Failed to resolve dataset from chart datasource\", error=str(e), payload={\"dataset_id\": dataset_id})\n\n unique_charts = {chart[\"id\"]: chart for chart in charts}\n unique_datasets = {dataset[\"id\"]: dataset for dataset in datasets}\n\n resolved_dashboard_id = dashboard_data.get(\"id\", dashboard_ref)\n return {\n \"id\": resolved_dashboard_id,\n \"title\": dashboard_data.get(\"dashboard_title\")\n or dashboard_data.get(\"title\") or f\"Dashboard {resolved_dashboard_id}\",\n \"slug\": dashboard_data.get(\"slug\"),\n \"url\": dashboard_data.get(\"url\"),\n \"description\": dashboard_data.get(\"description\") or \"\",\n \"last_modified\": dashboard_data.get(\"changed_on_utc\")\n or dashboard_data.get(\"changed_on\"),\n \"published\": dashboard_data.get(\"published\"),\n \"charts\": list(unique_charts.values()),\n \"datasets\": list(unique_datasets.values()),\n \"chart_count\": len(unique_charts),\n \"dataset_count\": len(unique_datasets),\n }\n\n # [/DEF:SupersetClientGetDashboardDetail:Function]\n\n # [DEF:SupersetClientExportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Экспортирует дашборд в виде ZIP-архива.\n # @SIDE_EFFECT: Performs network I/O to download archive.\n # @RELATION: CALLS -> [APIClient]\n def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:\n with belief_scope(\"export_dashboard\"):\n log.reason(\"Exporting dashboard\", payload={\"dashboard_id\": dashboard_id})\n response = self.network.request(\n method=\"GET\",\n endpoint=\"/dashboard/export/\",\n params={\"q\": json.dumps([dashboard_id])},\n stream=True,\n raw_response=True,\n )\n response = cast(Response, response)\n self._validate_export_response(response, dashboard_id)\n filename = self._resolve_export_filename(response, dashboard_id)\n log.reflect(\"Dashboard exported\", payload={\"dashboard_id\": dashboard_id, \"filename\": filename})\n return response.content, filename\n\n # [/DEF:SupersetClientExportDashboard:Function]\n\n # [DEF:SupersetClientImportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Импортирует дашборд из ZIP-файла.\n # @SIDE_EFFECT: Performs network I/O to upload archive.\n # @RELATION: CALLS -> [SupersetClientDoImport]\n # @RELATION: CALLS -> [APIClient]\n def import_dashboard(\n self,\n file_name: Union[str, Path],\n dash_id: Optional[int] = None,\n dash_slug: Optional[str] = None,\n ) -> Dict:\n with belief_scope(\"import_dashboard\"):\n if file_name is None:\n raise ValueError(\"file_name cannot be None\")\n file_path = str(file_name)\n self._validate_import_file(file_path)\n try:\n return self._do_import(file_path)\n except Exception as exc:\n log.explore(\"First import attempt failed\", error=str(exc), payload={\"file_name\": file_name})\n if not self.delete_before_reimport:\n raise\n\n target_id = self._resolve_target_id_for_delete(dash_id, dash_slug)\n if target_id is None:\n log.explore(\"No ID available for delete-retry during import\", error=\"Cannot delete-retry: neither dash_id nor dash_slug resolved\", payload={\"dash_id\": dash_id, \"dash_slug\": dash_slug})\n raise\n\n self.delete_dashboard(target_id)\n log.reason(\n \"Deleted dashboard, retrying import\",\n payload={\"target_id\": target_id},\n )\n return self._do_import(file_path)\n\n # [/DEF:SupersetClientImportDashboard:Function]\n\n # [DEF:SupersetClientDeleteDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Удаляет дашборд по его ID или slug.\n # @SIDE_EFFECT: Deletes resource from upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:\n with belief_scope(\"delete_dashboard\"):\n log.reason(\"Deleting dashboard\", payload={\"dashboard_id\": dashboard_id})\n response = self.network.request(\n method=\"DELETE\", endpoint=f\"/dashboard/{dashboard_id}\"\n )\n response = cast(Dict, response)\n if response.get(\"result\", True) is not False:\n log.reflect(\"Dashboard deleted\", payload={\"dashboard_id\": dashboard_id})\n else:\n log.explore(\"Unexpected response while deleting dashboard\", error=f\"Unexpected API response: {response}\", payload={\"dashboard_id\": dashboard_id, \"response\": response})\n\n # [/DEF:SupersetClientDeleteDashboard:Function]\n\n\n# [/DEF:SupersetDashboardsCrudMixin:Class]\n# [/DEF:SupersetDashboardsCrudMixin:Module]\n" }, { "contract_id": "SupersetClientGetDashboardDetail", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_crud.py", - "start_line": 28, - "end_line": 275, + "start_line": 29, + "end_line": 269, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39556,14 +40312,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDashboardDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dashboard information including related charts and datasets.\n # @RELATION: CALLS -> [SupersetClientGetDashboard]\n # @RELATION: CALLS -> [SupersetClientGetChart]\n def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:\n with belief_scope(\n \"SupersetClient.get_dashboard_detail\", f\"ref={dashboard_ref}\"\n ):\n dashboard_response = self.get_dashboard(dashboard_ref)\n dashboard_data = dashboard_response.get(\"result\", dashboard_response)\n\n charts: List[Dict] = []\n datasets: List[Dict] = []\n\n # [DEF:extract_dataset_id_from_form_data:Function]\n def extract_dataset_id_from_form_data(\n form_data: Optional[Dict],\n ) -> Optional[int]:\n if not isinstance(form_data, dict):\n return None\n datasource = form_data.get(\"datasource\")\n if isinstance(datasource, str):\n matched = re.match(r\"^(\\d+)__\", datasource)\n if matched:\n try:\n return int(matched.group(1))\n except ValueError:\n return None\n if isinstance(datasource, dict):\n ds_id = datasource.get(\"id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n ds_id = form_data.get(\"datasource_id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n\n # [/DEF:extract_dataset_id_from_form_data:Function]\n\n try:\n charts_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/charts\"\n )\n charts_payload = (\n charts_response.get(\"result\", [])\n if isinstance(charts_response, dict)\n else []\n )\n for chart_obj in charts_payload:\n if not isinstance(chart_obj, dict):\n continue\n chart_id = chart_obj.get(\"id\")\n if chart_id is None:\n continue\n form_data = chart_obj.get(\"form_data\")\n if isinstance(form_data, str):\n try:\n form_data = json.loads(form_data)\n except Exception:\n form_data = {}\n dataset_id = extract_dataset_id_from_form_data(\n form_data\n ) or chart_obj.get(\"datasource_id\")\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_obj.get(\"slice_name\")\n or chart_obj.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": (\n form_data.get(\"viz_type\")\n if isinstance(form_data, dict) else None\n ),\n \"dataset_id\": int(dataset_id) if dataset_id is not None else None,\n \"last_modified\": chart_obj.get(\"changed_on\"),\n \"overview\": chart_obj.get(\"description\")\n or (form_data.get(\"viz_type\") if isinstance(form_data, dict) else None)\n or \"Chart\",\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s\", e,\n )\n\n try:\n datasets_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/datasets\"\n )\n datasets_payload = (\n datasets_response.get(\"result\", [])\n if isinstance(datasets_response, dict)\n else []\n )\n for dataset_obj in datasets_payload:\n if not isinstance(dataset_obj, dict):\n continue\n dataset_id = dataset_obj.get(\"id\")\n if dataset_id is None:\n continue\n db_payload = dataset_obj.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_obj.get(\"table_name\")\n or dataset_obj.get(\"datasource_name\")\n or dataset_obj.get(\"name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_obj.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or dataset_obj.get(\"database_name\") or \"Unknown\",\n \"last_modified\": dataset_obj.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s\", e,\n )\n\n # Fallback: derive chart IDs from layout metadata\n if not charts:\n raw_position_json = dashboard_data.get(\"position_json\")\n chart_ids_from_position = set()\n if isinstance(raw_position_json, str) and raw_position_json:\n try:\n parsed_position = json.loads(raw_position_json)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_position)\n )\n except Exception:\n pass\n elif isinstance(raw_position_json, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_position_json)\n )\n\n raw_json_metadata = dashboard_data.get(\"json_metadata\")\n if isinstance(raw_json_metadata, str) and raw_json_metadata:\n try:\n parsed_metadata = json.loads(raw_json_metadata)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_metadata)\n )\n except Exception:\n pass\n elif isinstance(raw_json_metadata, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_json_metadata)\n )\n\n app_logger.info(\n \"[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)\",\n len(chart_ids_from_position), dashboard_ref,\n )\n\n for chart_id in sorted(chart_ids_from_position):\n try:\n chart_response = self.get_chart(int(chart_id))\n chart_data = chart_response.get(\"result\", chart_response)\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_data.get(\"slice_name\")\n or chart_data.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": chart_data.get(\"viz_type\"),\n \"dataset_id\": chart_data.get(\"datasource_id\"),\n \"last_modified\": chart_data.get(\"changed_on\"),\n \"overview\": chart_data.get(\"description\")\n or chart_data.get(\"viz_type\") or \"Chart\",\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s\",\n chart_id, e,\n )\n\n # Backfill datasets from chart datasource IDs.\n dataset_ids_from_charts = {\n c.get(\"dataset_id\") for c in charts if c.get(\"dataset_id\") is not None\n }\n known_dataset_ids = {\n d.get(\"id\") for d in datasets if d.get(\"id\") is not None\n }\n missing_dataset_ids: List[int] = []\n for raw_dataset_id in dataset_ids_from_charts:\n if raw_dataset_id is None or raw_dataset_id in known_dataset_ids:\n continue\n try:\n missing_dataset_ids.append(int(raw_dataset_id))\n except (TypeError, ValueError):\n continue\n\n for dataset_id in missing_dataset_ids:\n try:\n dataset_response = self.get_dataset(int(dataset_id))\n dataset_data = dataset_response.get(\"result\", dataset_response)\n db_payload = dataset_data.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_data.get(\"table_name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_data.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or \"Unknown\",\n \"last_modified\": dataset_data.get(\"changed_on_utc\")\n or dataset_data.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n app_logger.warning(\n \"[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s\",\n dataset_id, e,\n )\n\n unique_charts = {chart[\"id\"]: chart for chart in charts}\n unique_datasets = {dataset[\"id\"]: dataset for dataset in datasets}\n\n resolved_dashboard_id = dashboard_data.get(\"id\", dashboard_ref)\n return {\n \"id\": resolved_dashboard_id,\n \"title\": dashboard_data.get(\"dashboard_title\")\n or dashboard_data.get(\"title\") or f\"Dashboard {resolved_dashboard_id}\",\n \"slug\": dashboard_data.get(\"slug\"),\n \"url\": dashboard_data.get(\"url\"),\n \"description\": dashboard_data.get(\"description\") or \"\",\n \"last_modified\": dashboard_data.get(\"changed_on_utc\")\n or dashboard_data.get(\"changed_on\"),\n \"published\": dashboard_data.get(\"published\"),\n \"charts\": list(unique_charts.values()),\n \"datasets\": list(unique_datasets.values()),\n \"chart_count\": len(unique_charts),\n \"dataset_count\": len(unique_datasets),\n }\n\n # [/DEF:SupersetClientGetDashboardDetail:Function]\n" + "body": " # [DEF:SupersetClientGetDashboardDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dashboard information including related charts and datasets.\n # @RELATION: CALLS -> [SupersetClientGetDashboard]\n # @RELATION: CALLS -> [SupersetClientGetChart]\n def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:\n with belief_scope(\n \"SupersetClient.get_dashboard_detail\", f\"ref={dashboard_ref}\"\n ):\n dashboard_response = self.get_dashboard(dashboard_ref)\n dashboard_data = dashboard_response.get(\"result\", dashboard_response)\n\n charts: List[Dict] = []\n datasets: List[Dict] = []\n\n # [DEF:extract_dataset_id_from_form_data:Function]\n def extract_dataset_id_from_form_data(\n form_data: Optional[Dict],\n ) -> Optional[int]:\n if not isinstance(form_data, dict):\n return None\n datasource = form_data.get(\"datasource\")\n if isinstance(datasource, str):\n matched = re.match(r\"^(\\d+)__\", datasource)\n if matched:\n try:\n return int(matched.group(1))\n except ValueError:\n return None\n if isinstance(datasource, dict):\n ds_id = datasource.get(\"id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n ds_id = form_data.get(\"datasource_id\")\n try:\n return int(ds_id) if ds_id is not None else None\n except (TypeError, ValueError):\n return None\n\n # [/DEF:extract_dataset_id_from_form_data:Function]\n\n try:\n charts_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/charts\"\n )\n charts_payload = (\n charts_response.get(\"result\", [])\n if isinstance(charts_response, dict)\n else []\n )\n for chart_obj in charts_payload:\n if not isinstance(chart_obj, dict):\n continue\n chart_id = chart_obj.get(\"id\")\n if chart_id is None:\n continue\n form_data = chart_obj.get(\"form_data\")\n if isinstance(form_data, str):\n try:\n form_data = json.loads(form_data)\n except Exception:\n form_data = {}\n dataset_id = extract_dataset_id_from_form_data(\n form_data\n ) or chart_obj.get(\"datasource_id\")\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_obj.get(\"slice_name\")\n or chart_obj.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": (\n form_data.get(\"viz_type\")\n if isinstance(form_data, dict) else None\n ),\n \"dataset_id\": int(dataset_id) if dataset_id is not None else None,\n \"last_modified\": chart_obj.get(\"changed_on\"),\n \"overview\": chart_obj.get(\"description\")\n or (form_data.get(\"viz_type\") if isinstance(form_data, dict) else None)\n or \"Chart\",\n })\n except Exception as e:\n log.explore(\"Failed to fetch dashboard charts\", error=str(e), payload={\"dashboard_ref\": dashboard_ref})\n\n try:\n datasets_response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}/datasets\"\n )\n datasets_payload = (\n datasets_response.get(\"result\", [])\n if isinstance(datasets_response, dict)\n else []\n )\n for dataset_obj in datasets_payload:\n if not isinstance(dataset_obj, dict):\n continue\n dataset_id = dataset_obj.get(\"id\")\n if dataset_id is None:\n continue\n db_payload = dataset_obj.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_obj.get(\"table_name\")\n or dataset_obj.get(\"datasource_name\")\n or dataset_obj.get(\"name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_obj.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or dataset_obj.get(\"database_name\") or \"Unknown\",\n \"last_modified\": dataset_obj.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n log.explore(\"Failed to fetch dashboard datasets\", error=str(e), payload={\"dashboard_ref\": dashboard_ref})\n\n # Fallback: derive chart IDs from layout metadata\n if not charts:\n raw_position_json = dashboard_data.get(\"position_json\")\n chart_ids_from_position = set()\n if isinstance(raw_position_json, str) and raw_position_json:\n try:\n parsed_position = json.loads(raw_position_json)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_position)\n )\n except Exception:\n pass\n elif isinstance(raw_position_json, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_position_json)\n )\n\n raw_json_metadata = dashboard_data.get(\"json_metadata\")\n if isinstance(raw_json_metadata, str) and raw_json_metadata:\n try:\n parsed_metadata = json.loads(raw_json_metadata)\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(parsed_metadata)\n )\n except Exception:\n pass\n elif isinstance(raw_json_metadata, dict):\n chart_ids_from_position.update(\n self._extract_chart_ids_from_layout(raw_json_metadata)\n )\n\n log.reason(\n \"Extracted fallback chart IDs from layout\",\n payload={\n \"chart_count\": len(chart_ids_from_position),\n \"dashboard_ref\": dashboard_ref,\n },\n )\n\n for chart_id in sorted(chart_ids_from_position):\n try:\n chart_response = self.get_chart(int(chart_id))\n chart_data = chart_response.get(\"result\", chart_response)\n charts.append({\n \"id\": int(chart_id),\n \"title\": chart_data.get(\"slice_name\")\n or chart_data.get(\"name\") or f\"Chart {chart_id}\",\n \"viz_type\": chart_data.get(\"viz_type\"),\n \"dataset_id\": chart_data.get(\"datasource_id\"),\n \"last_modified\": chart_data.get(\"changed_on\"),\n \"overview\": chart_data.get(\"description\")\n or chart_data.get(\"viz_type\") or \"Chart\",\n })\n except Exception as e:\n log.explore(\"Failed to resolve fallback chart\", error=str(e), payload={\"chart_id\": chart_id})\n\n # Backfill datasets from chart datasource IDs.\n dataset_ids_from_charts = {\n c.get(\"dataset_id\") for c in charts if c.get(\"dataset_id\") is not None\n }\n known_dataset_ids = {\n d.get(\"id\") for d in datasets if d.get(\"id\") is not None\n }\n missing_dataset_ids: List[int] = []\n for raw_dataset_id in dataset_ids_from_charts:\n if raw_dataset_id is None or raw_dataset_id in known_dataset_ids:\n continue\n try:\n missing_dataset_ids.append(int(raw_dataset_id))\n except (TypeError, ValueError):\n continue\n\n for dataset_id in missing_dataset_ids:\n try:\n dataset_response = self.get_dataset(int(dataset_id))\n dataset_data = dataset_response.get(\"result\", dataset_response)\n db_payload = dataset_data.get(\"database\")\n db_name = (\n db_payload.get(\"database_name\")\n if isinstance(db_payload, dict) else None\n )\n table_name = (\n dataset_data.get(\"table_name\") or f\"Dataset {dataset_id}\"\n )\n schema = dataset_data.get(\"schema\")\n fq_name = f\"{schema}.{table_name}\" if schema else table_name\n datasets.append({\n \"id\": int(dataset_id),\n \"table_name\": table_name,\n \"schema\": schema,\n \"database\": db_name or \"Unknown\",\n \"last_modified\": dataset_data.get(\"changed_on_utc\")\n or dataset_data.get(\"changed_on\"),\n \"overview\": fq_name,\n })\n except Exception as e:\n log.explore(\"Failed to resolve dataset from chart datasource\", error=str(e), payload={\"dataset_id\": dataset_id})\n\n unique_charts = {chart[\"id\"]: chart for chart in charts}\n unique_datasets = {dataset[\"id\"]: dataset for dataset in datasets}\n\n resolved_dashboard_id = dashboard_data.get(\"id\", dashboard_ref)\n return {\n \"id\": resolved_dashboard_id,\n \"title\": dashboard_data.get(\"dashboard_title\")\n or dashboard_data.get(\"title\") or f\"Dashboard {resolved_dashboard_id}\",\n \"slug\": dashboard_data.get(\"slug\"),\n \"url\": dashboard_data.get(\"url\"),\n \"description\": dashboard_data.get(\"description\") or \"\",\n \"last_modified\": dashboard_data.get(\"changed_on_utc\")\n or dashboard_data.get(\"changed_on\"),\n \"published\": dashboard_data.get(\"published\"),\n \"charts\": list(unique_charts.values()),\n \"datasets\": list(unique_datasets.values()),\n \"chart_count\": len(unique_charts),\n \"dataset_count\": len(unique_datasets),\n }\n\n # [/DEF:SupersetClientGetDashboardDetail:Function]\n" }, { "contract_id": "extract_dataset_id_from_form_data", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_crud.py", - "start_line": 43, - "end_line": 69, + "start_line": 44, + "end_line": 70, "tier": "TIER_1", "complexity": 1, "metadata": {}, @@ -39586,8 +40342,8 @@ "contract_id": "SupersetClientExportDashboard", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_crud.py", - "start_line": 277, - "end_line": 303, + "start_line": 271, + "end_line": 292, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39615,14 +40371,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientExportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Экспортирует дашборд в виде ZIP-архива.\n # @SIDE_EFFECT: Performs network I/O to download archive.\n # @RELATION: CALLS -> [APIClient]\n def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:\n with belief_scope(\"export_dashboard\"):\n app_logger.info(\n \"[export_dashboard][Enter] Exporting dashboard %s.\", dashboard_id\n )\n response = self.network.request(\n method=\"GET\",\n endpoint=\"/dashboard/export/\",\n params={\"q\": json.dumps([dashboard_id])},\n stream=True,\n raw_response=True,\n )\n response = cast(Response, response)\n self._validate_export_response(response, dashboard_id)\n filename = self._resolve_export_filename(response, dashboard_id)\n app_logger.info(\n \"[export_dashboard][Exit] Exported dashboard %s to %s.\",\n dashboard_id, filename,\n )\n return response.content, filename\n\n # [/DEF:SupersetClientExportDashboard:Function]\n" + "body": " # [DEF:SupersetClientExportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Экспортирует дашборд в виде ZIP-архива.\n # @SIDE_EFFECT: Performs network I/O to download archive.\n # @RELATION: CALLS -> [APIClient]\n def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:\n with belief_scope(\"export_dashboard\"):\n log.reason(\"Exporting dashboard\", payload={\"dashboard_id\": dashboard_id})\n response = self.network.request(\n method=\"GET\",\n endpoint=\"/dashboard/export/\",\n params={\"q\": json.dumps([dashboard_id])},\n stream=True,\n raw_response=True,\n )\n response = cast(Response, response)\n self._validate_export_response(response, dashboard_id)\n filename = self._resolve_export_filename(response, dashboard_id)\n log.reflect(\"Dashboard exported\", payload={\"dashboard_id\": dashboard_id, \"filename\": filename})\n return response.content, filename\n\n # [/DEF:SupersetClientExportDashboard:Function]\n" }, { "contract_id": "SupersetClientImportDashboard", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_crud.py", - "start_line": 305, - "end_line": 346, + "start_line": 294, + "end_line": 330, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39656,14 +40412,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientImportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Импортирует дашборд из ZIP-файла.\n # @SIDE_EFFECT: Performs network I/O to upload archive.\n # @RELATION: CALLS -> [SupersetClientDoImport]\n # @RELATION: CALLS -> [APIClient]\n def import_dashboard(\n self,\n file_name: Union[str, Path],\n dash_id: Optional[int] = None,\n dash_slug: Optional[str] = None,\n ) -> Dict:\n with belief_scope(\"import_dashboard\"):\n if file_name is None:\n raise ValueError(\"file_name cannot be None\")\n file_path = str(file_name)\n self._validate_import_file(file_path)\n try:\n return self._do_import(file_path)\n except Exception as exc:\n app_logger.error(\n \"[import_dashboard][Failure] First import attempt failed: %s\",\n exc, exc_info=True,\n )\n if not self.delete_before_reimport:\n raise\n\n target_id = self._resolve_target_id_for_delete(dash_id, dash_slug)\n if target_id is None:\n app_logger.error(\n \"[import_dashboard][Failure] No ID available for delete-retry.\"\n )\n raise\n\n self.delete_dashboard(target_id)\n app_logger.info(\n \"[import_dashboard][State] Deleted dashboard ID %s, retrying import.\",\n target_id,\n )\n return self._do_import(file_path)\n\n # [/DEF:SupersetClientImportDashboard:Function]\n" + "body": " # [DEF:SupersetClientImportDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Импортирует дашборд из ZIP-файла.\n # @SIDE_EFFECT: Performs network I/O to upload archive.\n # @RELATION: CALLS -> [SupersetClientDoImport]\n # @RELATION: CALLS -> [APIClient]\n def import_dashboard(\n self,\n file_name: Union[str, Path],\n dash_id: Optional[int] = None,\n dash_slug: Optional[str] = None,\n ) -> Dict:\n with belief_scope(\"import_dashboard\"):\n if file_name is None:\n raise ValueError(\"file_name cannot be None\")\n file_path = str(file_name)\n self._validate_import_file(file_path)\n try:\n return self._do_import(file_path)\n except Exception as exc:\n log.explore(\"First import attempt failed\", error=str(exc), payload={\"file_name\": file_name})\n if not self.delete_before_reimport:\n raise\n\n target_id = self._resolve_target_id_for_delete(dash_id, dash_slug)\n if target_id is None:\n log.explore(\"No ID available for delete-retry during import\", error=\"Cannot delete-retry: neither dash_id nor dash_slug resolved\", payload={\"dash_id\": dash_id, \"dash_slug\": dash_slug})\n raise\n\n self.delete_dashboard(target_id)\n log.reason(\n \"Deleted dashboard, retrying import\",\n payload={\"target_id\": target_id},\n )\n return self._do_import(file_path)\n\n # [/DEF:SupersetClientImportDashboard:Function]\n" }, { "contract_id": "SupersetClientDeleteDashboard", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_crud.py", - "start_line": 348, - "end_line": 372, + "start_line": 332, + "end_line": 349, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39691,14 +40447,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientDeleteDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Удаляет дашборд по его ID или slug.\n # @SIDE_EFFECT: Deletes resource from upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:\n with belief_scope(\"delete_dashboard\"):\n app_logger.info(\n \"[delete_dashboard][Enter] Deleting dashboard %s.\", dashboard_id\n )\n response = self.network.request(\n method=\"DELETE\", endpoint=f\"/dashboard/{dashboard_id}\"\n )\n response = cast(Dict, response)\n if response.get(\"result\", True) is not False:\n app_logger.info(\n \"[delete_dashboard][Success] Dashboard %s deleted.\", dashboard_id\n )\n else:\n app_logger.warning(\n \"[delete_dashboard][Warning] Unexpected response while deleting %s: %s\",\n dashboard_id, response,\n )\n\n # [/DEF:SupersetClientDeleteDashboard:Function]\n" + "body": " # [DEF:SupersetClientDeleteDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Удаляет дашборд по его ID или slug.\n # @SIDE_EFFECT: Deletes resource from upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:\n with belief_scope(\"delete_dashboard\"):\n log.reason(\"Deleting dashboard\", payload={\"dashboard_id\": dashboard_id})\n response = self.network.request(\n method=\"DELETE\", endpoint=f\"/dashboard/{dashboard_id}\"\n )\n response = cast(Dict, response)\n if response.get(\"result\", True) is not False:\n log.reflect(\"Dashboard deleted\", payload={\"dashboard_id\": dashboard_id})\n else:\n log.explore(\"Unexpected response while deleting dashboard\", error=f\"Unexpected API response: {response}\", payload={\"dashboard_id\": dashboard_id, \"response\": response})\n\n # [/DEF:SupersetClientDeleteDashboard:Function]\n" }, { "contract_id": "SupersetDashboardsFiltersMixin", "contract_type": "Module", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", "start_line": 1, - "end_line": 268, + "end_line": 257, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39723,14 +40479,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetDashboardsFiltersMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, dashboards, filters, permalink, native-filters\n# @PURPOSE: Dashboard native filter extraction mixin for SupersetClient.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n\nimport json\nfrom typing import Any, Dict, Optional, Union, cast\n\nfrom ..logger import logger as app_logger, belief_scope\n\napp_logger = cast(Any, app_logger)\n\n\n# [DEF:SupersetDashboardsFiltersMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing dashboard native filter extraction from permalink and URL state.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\nclass SupersetDashboardsFiltersMixin:\n # [DEF:SupersetClientGetDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches a single dashboard by ID or slug.\n # @RELATION: CALLS -> [APIClient]\n def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict:\n with belief_scope(\"SupersetClient.get_dashboard\", f\"ref={dashboard_ref}\"):\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}\"\n )\n return cast(Dict, response)\n\n # [/DEF:SupersetClientGetDashboard:Function]\n\n # [DEF:SupersetClientGetDashboardPermalinkState:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Fetches stored dashboard permalink state by permalink key.\n # @RELATION: CALLS -> [APIClient]\n def get_dashboard_permalink_state(self, permalink_key: str) -> Dict:\n with belief_scope(\n \"SupersetClient.get_dashboard_permalink_state\", f\"key={permalink_key}\"\n ):\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/permalink/{permalink_key}\"\n )\n return cast(Dict, response)\n\n # [/DEF:SupersetClientGetDashboardPermalinkState:Function]\n\n # [DEF:SupersetClientGetNativeFilterState:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Fetches stored native filter state by filter state key.\n # @RELATION: CALLS -> [APIClient]\n def get_native_filter_state(\n self, dashboard_id: Union[int, str], filter_state_key: str\n ) -> Dict:\n with belief_scope(\n \"SupersetClient.get_native_filter_state\",\n f\"dashboard={dashboard_id}, key={filter_state_key}\",\n ):\n response = self.network.request(\n method=\"GET\",\n endpoint=f\"/dashboard/{dashboard_id}/filter_state/{filter_state_key}\",\n )\n return cast(Dict, response)\n\n # [/DEF:SupersetClientGetNativeFilterState:Function]\n\n # [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract native filters dataMask from a permalink key.\n # @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState]\n def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict:\n with belief_scope(\n \"SupersetClient.extract_native_filters_from_permalink\",\n f\"key={permalink_key}\",\n ):\n permalink_response = self.get_dashboard_permalink_state(permalink_key)\n\n result = permalink_response.get(\"result\", permalink_response)\n state = result.get(\"state\", result)\n data_mask = state.get(\"dataMask\", {})\n\n extracted_filters = {}\n for filter_id, filter_data in data_mask.items():\n if not isinstance(filter_data, dict):\n continue\n extracted_filters[filter_id] = {\n \"extraFormData\": filter_data.get(\"extraFormData\", {}),\n \"filterState\": filter_data.get(\"filterState\", {}),\n \"ownState\": filter_data.get(\"ownState\", {}),\n }\n\n return {\n \"dataMask\": extracted_filters,\n \"activeTabs\": state.get(\"activeTabs\", []),\n \"anchor\": state.get(\"anchor\"),\n \"chartStates\": state.get(\"chartStates\", {}),\n \"permalink_key\": permalink_key,\n }\n\n # [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]\n\n # [DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract native filters from a native_filters_key URL parameter.\n # @RELATION: CALLS -> [SupersetClientGetNativeFilterState]\n def extract_native_filters_from_key(\n self, dashboard_id: Union[int, str], filter_state_key: str\n ) -> Dict:\n with belief_scope(\n \"SupersetClient.extract_native_filters_from_key\",\n f\"dashboard={dashboard_id}, key={filter_state_key}\",\n ):\n filter_response = self.get_native_filter_state(\n dashboard_id, filter_state_key\n )\n\n result = filter_response.get(\"result\", filter_response)\n value = result.get(\"value\")\n\n if isinstance(value, str):\n try:\n parsed_value = json.loads(value)\n except json.JSONDecodeError as e:\n app_logger.warning(\n \"[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s\",\n e,\n )\n parsed_value = {}\n elif isinstance(value, dict):\n parsed_value = value\n else:\n parsed_value = {}\n\n extracted_filters = {}\n\n if \"id\" in parsed_value and \"extraFormData\" in parsed_value:\n filter_id = parsed_value.get(\"id\", filter_state_key)\n extracted_filters[filter_id] = {\n \"extraFormData\": parsed_value.get(\"extraFormData\", {}),\n \"filterState\": parsed_value.get(\"filterState\", {}),\n \"ownState\": parsed_value.get(\"ownState\", {}),\n }\n else:\n for filter_id, filter_data in parsed_value.items():\n if not isinstance(filter_data, dict):\n continue\n extracted_filters[filter_id] = {\n \"extraFormData\": filter_data.get(\"extraFormData\", {}),\n \"filterState\": filter_data.get(\"filterState\", {}),\n \"ownState\": filter_data.get(\"ownState\", {}),\n }\n\n return {\n \"dataMask\": extracted_filters,\n \"dashboard_id\": dashboard_id,\n \"filter_state_key\": filter_state_key,\n }\n\n # [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n\n # [DEF:SupersetClientParseDashboardUrlForFilters:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]\n def parse_dashboard_url_for_filters(self, url: str) -> Dict:\n with belief_scope(\n \"SupersetClient.parse_dashboard_url_for_filters\", f\"url={url}\"\n ):\n import urllib.parse\n\n parsed_url = urllib.parse.urlparse(url)\n query_params = urllib.parse.parse_qs(parsed_url.query)\n path_parts = parsed_url.path.rstrip(\"/\").split(\"/\")\n\n result = {\n \"url\": url,\n \"dashboard_id\": None,\n \"filter_type\": None,\n \"filters\": {},\n }\n\n # Check for permalink URL: /dashboard/p/{key}/ or /superset/dashboard/p/{key}/\n if \"p\" in path_parts:\n try:\n p_index = path_parts.index(\"p\")\n if p_index + 1 < len(path_parts):\n permalink_key = path_parts[p_index + 1]\n filter_data = self.extract_native_filters_from_permalink(\n permalink_key\n )\n result[\"filter_type\"] = \"permalink\"\n result[\"filters\"] = filter_data\n return result\n except ValueError:\n pass\n\n # Check for native_filters_key in query params\n native_filters_key = query_params.get(\"native_filters_key\", [None])[0]\n if native_filters_key:\n dashboard_ref = None\n if \"dashboard\" in path_parts:\n try:\n dash_index = path_parts.index(\"dashboard\")\n if dash_index + 1 < len(path_parts):\n potential_id = path_parts[dash_index + 1]\n if potential_id not in (\"p\", \"list\", \"new\"):\n dashboard_ref = potential_id\n except ValueError:\n pass\n\n if dashboard_ref:\n resolved_id = None\n try:\n resolved_id = int(dashboard_ref)\n except (ValueError, TypeError):\n try:\n dash_resp = self.get_dashboard(dashboard_ref)\n dash_data = (\n dash_resp.get(\"result\", dash_resp)\n if isinstance(dash_resp, dict)\n else {}\n )\n raw_id = dash_data.get(\"id\")\n if raw_id is not None:\n resolved_id = int(raw_id)\n except Exception as e:\n app_logger.warning(\n \"[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s\",\n dashboard_ref,\n e,\n )\n\n if resolved_id is not None:\n filter_data = self.extract_native_filters_from_key(\n resolved_id, native_filters_key\n )\n result[\"filter_type\"] = \"native_filters_key\"\n result[\"dashboard_id\"] = resolved_id\n result[\"filters\"] = filter_data\n return result\n else:\n app_logger.warning(\n \"[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key\"\n )\n\n # Check for native_filters in query params (direct filter values)\n native_filters = query_params.get(\"native_filters\", [None])[0]\n if native_filters:\n try:\n parsed_filters = json.loads(native_filters)\n result[\"filter_type\"] = \"native_filters\"\n result[\"filters\"] = {\"dataMask\": parsed_filters}\n return result\n except json.JSONDecodeError as e:\n app_logger.warning(\n \"[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s\",\n e,\n )\n\n return result\n\n # [/DEF:SupersetClientParseDashboardUrlForFilters:Function]\n\n\n# [/DEF:SupersetDashboardsFiltersMixin:Class]\n# [/DEF:SupersetDashboardsFiltersMixin:Module]\n" + "body": "# [DEF:SupersetDashboardsFiltersMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, dashboards, filters, permalink, native-filters\n# @PURPOSE: Dashboard native filter extraction mixin for SupersetClient.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n\nimport json\nfrom typing import Any, Dict, Optional, Union, cast\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\n\nlog = MarkerLogger(\"SupersetDashboardsFilters\")\n\n\n# [DEF:SupersetDashboardsFiltersMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing dashboard native filter extraction from permalink and URL state.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\nclass SupersetDashboardsFiltersMixin:\n # [DEF:SupersetClientGetDashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches a single dashboard by ID or slug.\n # @RELATION: CALLS -> [APIClient]\n def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict:\n with belief_scope(\"SupersetClient.get_dashboard\", f\"ref={dashboard_ref}\"):\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/{dashboard_ref}\"\n )\n return cast(Dict, response)\n\n # [/DEF:SupersetClientGetDashboard:Function]\n\n # [DEF:SupersetClientGetDashboardPermalinkState:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Fetches stored dashboard permalink state by permalink key.\n # @RELATION: CALLS -> [APIClient]\n def get_dashboard_permalink_state(self, permalink_key: str) -> Dict:\n with belief_scope(\n \"SupersetClient.get_dashboard_permalink_state\", f\"key={permalink_key}\"\n ):\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dashboard/permalink/{permalink_key}\"\n )\n return cast(Dict, response)\n\n # [/DEF:SupersetClientGetDashboardPermalinkState:Function]\n\n # [DEF:SupersetClientGetNativeFilterState:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Fetches stored native filter state by filter state key.\n # @RELATION: CALLS -> [APIClient]\n def get_native_filter_state(\n self, dashboard_id: Union[int, str], filter_state_key: str\n ) -> Dict:\n with belief_scope(\n \"SupersetClient.get_native_filter_state\",\n f\"dashboard={dashboard_id}, key={filter_state_key}\",\n ):\n response = self.network.request(\n method=\"GET\",\n endpoint=f\"/dashboard/{dashboard_id}/filter_state/{filter_state_key}\",\n )\n return cast(Dict, response)\n\n # [/DEF:SupersetClientGetNativeFilterState:Function]\n\n # [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract native filters dataMask from a permalink key.\n # @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState]\n def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict:\n with belief_scope(\n \"SupersetClient.extract_native_filters_from_permalink\",\n f\"key={permalink_key}\",\n ):\n permalink_response = self.get_dashboard_permalink_state(permalink_key)\n\n result = permalink_response.get(\"result\", permalink_response)\n state = result.get(\"state\", result)\n data_mask = state.get(\"dataMask\", {})\n\n extracted_filters = {}\n for filter_id, filter_data in data_mask.items():\n if not isinstance(filter_data, dict):\n continue\n extracted_filters[filter_id] = {\n \"extraFormData\": filter_data.get(\"extraFormData\", {}),\n \"filterState\": filter_data.get(\"filterState\", {}),\n \"ownState\": filter_data.get(\"ownState\", {}),\n }\n\n return {\n \"dataMask\": extracted_filters,\n \"activeTabs\": state.get(\"activeTabs\", []),\n \"anchor\": state.get(\"anchor\"),\n \"chartStates\": state.get(\"chartStates\", {}),\n \"permalink_key\": permalink_key,\n }\n\n # [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]\n\n # [DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract native filters from a native_filters_key URL parameter.\n # @RELATION: CALLS -> [SupersetClientGetNativeFilterState]\n def extract_native_filters_from_key(\n self, dashboard_id: Union[int, str], filter_state_key: str\n ) -> Dict:\n with belief_scope(\n \"SupersetClient.extract_native_filters_from_key\",\n f\"dashboard={dashboard_id}, key={filter_state_key}\",\n ):\n filter_response = self.get_native_filter_state(\n dashboard_id, filter_state_key\n )\n\n result = filter_response.get(\"result\", filter_response)\n value = result.get(\"value\")\n\n if isinstance(value, str):\n try:\n parsed_value = json.loads(value)\n except json.JSONDecodeError as e:\n log.explore(\"Failed to parse filter state JSON\", error=str(e))\n parsed_value = {}\n elif isinstance(value, dict):\n parsed_value = value\n else:\n parsed_value = {}\n\n extracted_filters = {}\n\n if \"id\" in parsed_value and \"extraFormData\" in parsed_value:\n filter_id = parsed_value.get(\"id\", filter_state_key)\n extracted_filters[filter_id] = {\n \"extraFormData\": parsed_value.get(\"extraFormData\", {}),\n \"filterState\": parsed_value.get(\"filterState\", {}),\n \"ownState\": parsed_value.get(\"ownState\", {}),\n }\n else:\n for filter_id, filter_data in parsed_value.items():\n if not isinstance(filter_data, dict):\n continue\n extracted_filters[filter_id] = {\n \"extraFormData\": filter_data.get(\"extraFormData\", {}),\n \"filterState\": filter_data.get(\"filterState\", {}),\n \"ownState\": filter_data.get(\"ownState\", {}),\n }\n\n return {\n \"dataMask\": extracted_filters,\n \"dashboard_id\": dashboard_id,\n \"filter_state_key\": filter_state_key,\n }\n\n # [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n\n # [DEF:SupersetClientParseDashboardUrlForFilters:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]\n def parse_dashboard_url_for_filters(self, url: str) -> Dict:\n with belief_scope(\n \"SupersetClient.parse_dashboard_url_for_filters\", f\"url={url}\"\n ):\n import urllib.parse\n\n parsed_url = urllib.parse.urlparse(url)\n query_params = urllib.parse.parse_qs(parsed_url.query)\n path_parts = parsed_url.path.rstrip(\"/\").split(\"/\")\n\n result = {\n \"url\": url,\n \"dashboard_id\": None,\n \"filter_type\": None,\n \"filters\": {},\n }\n\n # Check for permalink URL: /dashboard/p/{key}/ or /superset/dashboard/p/{key}/\n if \"p\" in path_parts:\n try:\n p_index = path_parts.index(\"p\")\n if p_index + 1 < len(path_parts):\n permalink_key = path_parts[p_index + 1]\n filter_data = self.extract_native_filters_from_permalink(\n permalink_key\n )\n result[\"filter_type\"] = \"permalink\"\n result[\"filters\"] = filter_data\n return result\n except ValueError:\n pass\n\n # Check for native_filters_key in query params\n native_filters_key = query_params.get(\"native_filters_key\", [None])[0]\n if native_filters_key:\n dashboard_ref = None\n if \"dashboard\" in path_parts:\n try:\n dash_index = path_parts.index(\"dashboard\")\n if dash_index + 1 < len(path_parts):\n potential_id = path_parts[dash_index + 1]\n if potential_id not in (\"p\", \"list\", \"new\"):\n dashboard_ref = potential_id\n except ValueError:\n pass\n\n if dashboard_ref:\n resolved_id = None\n try:\n resolved_id = int(dashboard_ref)\n except (ValueError, TypeError):\n try:\n dash_resp = self.get_dashboard(dashboard_ref)\n dash_data = (\n dash_resp.get(\"result\", dash_resp)\n if isinstance(dash_resp, dict)\n else {}\n )\n raw_id = dash_data.get(\"id\")\n if raw_id is not None:\n resolved_id = int(raw_id)\n except Exception as e:\n log.explore(\"Failed to resolve dashboard slug to ID\", error=str(e), payload={\"slug\": dashboard_ref})\n\n if resolved_id is not None:\n filter_data = self.extract_native_filters_from_key(\n resolved_id, native_filters_key\n )\n result[\"filter_type\"] = \"native_filters_key\"\n result[\"dashboard_id\"] = resolved_id\n result[\"filters\"] = filter_data\n return result\n else:\n log.explore(\"Could not resolve dashboard_id from URL for native_filters_key\", error=\"Dashboard ID could not be resolved from URL\", payload={\"dashboard_ref\": dashboard_ref})\n\n # Check for native_filters in query params (direct filter values)\n native_filters = query_params.get(\"native_filters\", [None])[0]\n if native_filters:\n try:\n parsed_filters = json.loads(native_filters)\n result[\"filter_type\"] = \"native_filters\"\n result[\"filters\"] = {\"dataMask\": parsed_filters}\n return result\n except json.JSONDecodeError as e:\n log.explore(\"Failed to parse native_filters JSON\", error=str(e))\n\n return result\n\n # [/DEF:SupersetClientParseDashboardUrlForFilters:Function]\n\n\n# [/DEF:SupersetDashboardsFiltersMixin:Class]\n# [/DEF:SupersetDashboardsFiltersMixin:Module]\n" }, { "contract_id": "SupersetClientGetDashboard", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", - "start_line": 21, - "end_line": 32, + "start_line": 22, + "end_line": 33, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39753,8 +40509,8 @@ "contract_id": "SupersetClientGetDashboardPermalinkState", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", - "start_line": 34, - "end_line": 47, + "start_line": 35, + "end_line": 48, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -39787,8 +40543,8 @@ "contract_id": "SupersetClientGetNativeFilterState", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", - "start_line": 49, - "end_line": 66, + "start_line": 50, + "end_line": 67, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -39821,8 +40577,8 @@ "contract_id": "SupersetClientExtractNativeFiltersFromPermalink", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", - "start_line": 68, - "end_line": 101, + "start_line": 69, + "end_line": 102, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39845,8 +40601,8 @@ "contract_id": "SupersetClientExtractNativeFiltersFromKey", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", - "start_line": 103, - "end_line": 160, + "start_line": 104, + "end_line": 158, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39863,14 +40619,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract native filters from a native_filters_key URL parameter.\n # @RELATION: CALLS -> [SupersetClientGetNativeFilterState]\n def extract_native_filters_from_key(\n self, dashboard_id: Union[int, str], filter_state_key: str\n ) -> Dict:\n with belief_scope(\n \"SupersetClient.extract_native_filters_from_key\",\n f\"dashboard={dashboard_id}, key={filter_state_key}\",\n ):\n filter_response = self.get_native_filter_state(\n dashboard_id, filter_state_key\n )\n\n result = filter_response.get(\"result\", filter_response)\n value = result.get(\"value\")\n\n if isinstance(value, str):\n try:\n parsed_value = json.loads(value)\n except json.JSONDecodeError as e:\n app_logger.warning(\n \"[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s\",\n e,\n )\n parsed_value = {}\n elif isinstance(value, dict):\n parsed_value = value\n else:\n parsed_value = {}\n\n extracted_filters = {}\n\n if \"id\" in parsed_value and \"extraFormData\" in parsed_value:\n filter_id = parsed_value.get(\"id\", filter_state_key)\n extracted_filters[filter_id] = {\n \"extraFormData\": parsed_value.get(\"extraFormData\", {}),\n \"filterState\": parsed_value.get(\"filterState\", {}),\n \"ownState\": parsed_value.get(\"ownState\", {}),\n }\n else:\n for filter_id, filter_data in parsed_value.items():\n if not isinstance(filter_data, dict):\n continue\n extracted_filters[filter_id] = {\n \"extraFormData\": filter_data.get(\"extraFormData\", {}),\n \"filterState\": filter_data.get(\"filterState\", {}),\n \"ownState\": filter_data.get(\"ownState\", {}),\n }\n\n return {\n \"dataMask\": extracted_filters,\n \"dashboard_id\": dashboard_id,\n \"filter_state_key\": filter_state_key,\n }\n\n # [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n" + "body": " # [DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract native filters from a native_filters_key URL parameter.\n # @RELATION: CALLS -> [SupersetClientGetNativeFilterState]\n def extract_native_filters_from_key(\n self, dashboard_id: Union[int, str], filter_state_key: str\n ) -> Dict:\n with belief_scope(\n \"SupersetClient.extract_native_filters_from_key\",\n f\"dashboard={dashboard_id}, key={filter_state_key}\",\n ):\n filter_response = self.get_native_filter_state(\n dashboard_id, filter_state_key\n )\n\n result = filter_response.get(\"result\", filter_response)\n value = result.get(\"value\")\n\n if isinstance(value, str):\n try:\n parsed_value = json.loads(value)\n except json.JSONDecodeError as e:\n log.explore(\"Failed to parse filter state JSON\", error=str(e))\n parsed_value = {}\n elif isinstance(value, dict):\n parsed_value = value\n else:\n parsed_value = {}\n\n extracted_filters = {}\n\n if \"id\" in parsed_value and \"extraFormData\" in parsed_value:\n filter_id = parsed_value.get(\"id\", filter_state_key)\n extracted_filters[filter_id] = {\n \"extraFormData\": parsed_value.get(\"extraFormData\", {}),\n \"filterState\": parsed_value.get(\"filterState\", {}),\n \"ownState\": parsed_value.get(\"ownState\", {}),\n }\n else:\n for filter_id, filter_data in parsed_value.items():\n if not isinstance(filter_data, dict):\n continue\n extracted_filters[filter_id] = {\n \"extraFormData\": filter_data.get(\"extraFormData\", {}),\n \"filterState\": filter_data.get(\"filterState\", {}),\n \"ownState\": filter_data.get(\"ownState\", {}),\n }\n\n return {\n \"dataMask\": extracted_filters,\n \"dashboard_id\": dashboard_id,\n \"filter_state_key\": filter_state_key,\n }\n\n # [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]\n" }, { "contract_id": "SupersetClientParseDashboardUrlForFilters", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_filters.py", - "start_line": 162, - "end_line": 264, + "start_line": 160, + "end_line": 253, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39893,14 +40649,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientParseDashboardUrlForFilters:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]\n def parse_dashboard_url_for_filters(self, url: str) -> Dict:\n with belief_scope(\n \"SupersetClient.parse_dashboard_url_for_filters\", f\"url={url}\"\n ):\n import urllib.parse\n\n parsed_url = urllib.parse.urlparse(url)\n query_params = urllib.parse.parse_qs(parsed_url.query)\n path_parts = parsed_url.path.rstrip(\"/\").split(\"/\")\n\n result = {\n \"url\": url,\n \"dashboard_id\": None,\n \"filter_type\": None,\n \"filters\": {},\n }\n\n # Check for permalink URL: /dashboard/p/{key}/ or /superset/dashboard/p/{key}/\n if \"p\" in path_parts:\n try:\n p_index = path_parts.index(\"p\")\n if p_index + 1 < len(path_parts):\n permalink_key = path_parts[p_index + 1]\n filter_data = self.extract_native_filters_from_permalink(\n permalink_key\n )\n result[\"filter_type\"] = \"permalink\"\n result[\"filters\"] = filter_data\n return result\n except ValueError:\n pass\n\n # Check for native_filters_key in query params\n native_filters_key = query_params.get(\"native_filters_key\", [None])[0]\n if native_filters_key:\n dashboard_ref = None\n if \"dashboard\" in path_parts:\n try:\n dash_index = path_parts.index(\"dashboard\")\n if dash_index + 1 < len(path_parts):\n potential_id = path_parts[dash_index + 1]\n if potential_id not in (\"p\", \"list\", \"new\"):\n dashboard_ref = potential_id\n except ValueError:\n pass\n\n if dashboard_ref:\n resolved_id = None\n try:\n resolved_id = int(dashboard_ref)\n except (ValueError, TypeError):\n try:\n dash_resp = self.get_dashboard(dashboard_ref)\n dash_data = (\n dash_resp.get(\"result\", dash_resp)\n if isinstance(dash_resp, dict)\n else {}\n )\n raw_id = dash_data.get(\"id\")\n if raw_id is not None:\n resolved_id = int(raw_id)\n except Exception as e:\n app_logger.warning(\n \"[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s\",\n dashboard_ref,\n e,\n )\n\n if resolved_id is not None:\n filter_data = self.extract_native_filters_from_key(\n resolved_id, native_filters_key\n )\n result[\"filter_type\"] = \"native_filters_key\"\n result[\"dashboard_id\"] = resolved_id\n result[\"filters\"] = filter_data\n return result\n else:\n app_logger.warning(\n \"[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key\"\n )\n\n # Check for native_filters in query params (direct filter values)\n native_filters = query_params.get(\"native_filters\", [None])[0]\n if native_filters:\n try:\n parsed_filters = json.loads(native_filters)\n result[\"filter_type\"] = \"native_filters\"\n result[\"filters\"] = {\"dataMask\": parsed_filters}\n return result\n except json.JSONDecodeError as e:\n app_logger.warning(\n \"[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s\",\n e,\n )\n\n return result\n\n # [/DEF:SupersetClientParseDashboardUrlForFilters:Function]\n" + "body": " # [DEF:SupersetClientParseDashboardUrlForFilters:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]\n # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]\n def parse_dashboard_url_for_filters(self, url: str) -> Dict:\n with belief_scope(\n \"SupersetClient.parse_dashboard_url_for_filters\", f\"url={url}\"\n ):\n import urllib.parse\n\n parsed_url = urllib.parse.urlparse(url)\n query_params = urllib.parse.parse_qs(parsed_url.query)\n path_parts = parsed_url.path.rstrip(\"/\").split(\"/\")\n\n result = {\n \"url\": url,\n \"dashboard_id\": None,\n \"filter_type\": None,\n \"filters\": {},\n }\n\n # Check for permalink URL: /dashboard/p/{key}/ or /superset/dashboard/p/{key}/\n if \"p\" in path_parts:\n try:\n p_index = path_parts.index(\"p\")\n if p_index + 1 < len(path_parts):\n permalink_key = path_parts[p_index + 1]\n filter_data = self.extract_native_filters_from_permalink(\n permalink_key\n )\n result[\"filter_type\"] = \"permalink\"\n result[\"filters\"] = filter_data\n return result\n except ValueError:\n pass\n\n # Check for native_filters_key in query params\n native_filters_key = query_params.get(\"native_filters_key\", [None])[0]\n if native_filters_key:\n dashboard_ref = None\n if \"dashboard\" in path_parts:\n try:\n dash_index = path_parts.index(\"dashboard\")\n if dash_index + 1 < len(path_parts):\n potential_id = path_parts[dash_index + 1]\n if potential_id not in (\"p\", \"list\", \"new\"):\n dashboard_ref = potential_id\n except ValueError:\n pass\n\n if dashboard_ref:\n resolved_id = None\n try:\n resolved_id = int(dashboard_ref)\n except (ValueError, TypeError):\n try:\n dash_resp = self.get_dashboard(dashboard_ref)\n dash_data = (\n dash_resp.get(\"result\", dash_resp)\n if isinstance(dash_resp, dict)\n else {}\n )\n raw_id = dash_data.get(\"id\")\n if raw_id is not None:\n resolved_id = int(raw_id)\n except Exception as e:\n log.explore(\"Failed to resolve dashboard slug to ID\", error=str(e), payload={\"slug\": dashboard_ref})\n\n if resolved_id is not None:\n filter_data = self.extract_native_filters_from_key(\n resolved_id, native_filters_key\n )\n result[\"filter_type\"] = \"native_filters_key\"\n result[\"dashboard_id\"] = resolved_id\n result[\"filters\"] = filter_data\n return result\n else:\n log.explore(\"Could not resolve dashboard_id from URL for native_filters_key\", error=\"Dashboard ID could not be resolved from URL\", payload={\"dashboard_ref\": dashboard_ref})\n\n # Check for native_filters in query params (direct filter values)\n native_filters = query_params.get(\"native_filters\", [None])[0]\n if native_filters:\n try:\n parsed_filters = json.loads(native_filters)\n result[\"filter_type\"] = \"native_filters\"\n result[\"filters\"] = {\"dataMask\": parsed_filters}\n return result\n except json.JSONDecodeError as e:\n log.explore(\"Failed to parse native_filters JSON\", error=str(e))\n\n return result\n\n # [/DEF:SupersetClientParseDashboardUrlForFilters:Function]\n" }, { "contract_id": "SupersetDashboardsListMixin", "contract_type": "Module", "file_path": "backend/src/core/superset_client/_dashboards_list.py", "start_line": 1, - "end_line": 207, + "end_line": 218, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39931,14 +40687,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetDashboardsListMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, dashboards, list, pagination, summary\n# @PURPOSE: Dashboard listing mixin for SupersetClient — paginated list, summary projection.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]\n\nimport json\nfrom typing import Any, Dict, List, Optional, Tuple, cast\n\nfrom ..logger import logger as app_logger, belief_scope\n\napp_logger = cast(Any, app_logger)\n\n\n# [DEF:SupersetDashboardsListMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing dashboard listing and summary projection operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]\nclass SupersetDashboardsListMixin:\n # [DEF:SupersetClientGetDashboards:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_dashboards\"):\n app_logger.info(\"[get_dashboards][Enter] Fetching dashboards.\")\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = [\n \"slug\", \"id\", \"url\", \"changed_on_utc\", \"dashboard_title\",\n \"published\", \"created_by\", \"changed_by\", \"changed_by_name\", \"owners\",\n ]\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dashboard/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n app_logger.info(\"[get_dashboards][Exit] Found %d dashboards.\", total_count)\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDashboards:Function]\n\n # [DEF:SupersetClientGetDashboardsPage:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.\n # @RELATION: CALLS -> [APIClient]\n def get_dashboards_page(\n self, query: Optional[Dict] = None\n ) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_dashboards_page\"):\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = [\n \"slug\", \"id\", \"url\", \"changed_on_utc\", \"dashboard_title\",\n \"published\", \"created_by\", \"changed_by\", \"changed_by_name\", \"owners\",\n ]\n\n response_json = cast(\n Dict[str, Any],\n self.network.request(\n method=\"GET\",\n endpoint=\"/dashboard/\",\n params={\"q\": json.dumps(validated_query)},\n ),\n )\n result = response_json.get(\"result\", [])\n total_count = response_json.get(\"count\", len(result))\n return total_count, result\n\n # [/DEF:SupersetClientGetDashboardsPage:Function]\n\n # [DEF:SupersetClientGetDashboardsSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches dashboard metadata optimized for the grid.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_dashboards_summary\"):\n query: Dict[str, Any] = {}\n if require_slug:\n query[\"filters\"] = [{\"col\": \"slug\", \"opr\": \"neq\", \"value\": \"\"}]\n _, dashboards = self.get_dashboards(query=query)\n\n result = []\n max_debug_samples = 12\n for index, dash in enumerate(dashboards):\n raw_owners = dash.get(\"owners\")\n raw_created_by = dash.get(\"created_by\")\n raw_changed_by = dash.get(\"changed_by\")\n raw_changed_by_name = dash.get(\"changed_by_name\")\n\n owners = self._extract_owner_labels(raw_owners)\n if not owners:\n owners = self._extract_owner_labels([raw_created_by, raw_changed_by])\n\n projected_created_by = self._extract_user_display(None, raw_created_by)\n projected_modified_by = self._extract_user_display(\n raw_changed_by_name, raw_changed_by,\n )\n\n raw_owner_usernames: List[str] = []\n if isinstance(raw_owners, list):\n for owner_payload in raw_owners:\n if isinstance(owner_payload, dict):\n owner_username = self._sanitize_user_text(\n owner_payload.get(\"username\")\n )\n if owner_username:\n raw_owner_usernames.append(owner_username)\n\n result.append({\n \"id\": dash.get(\"id\"),\n \"slug\": dash.get(\"slug\"),\n \"title\": dash.get(\"dashboard_title\"),\n \"url\": dash.get(\"url\"),\n \"last_modified\": dash.get(\"changed_on_utc\"),\n \"status\": \"published\" if dash.get(\"published\") else \"draft\",\n \"created_by\": projected_created_by,\n \"modified_by\": projected_modified_by,\n \"owners\": owners,\n })\n\n if index < max_debug_samples:\n app_logger.reflect(\n \"[REFLECT] Dashboard actor projection sample \"\n f\"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, \"\n f\"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, \"\n f\"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, \"\n f\"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, \"\n f\"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})\"\n )\n\n app_logger.reflect(\n \"[REFLECT] Dashboard actor projection summary \"\n f\"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, \"\n f\"sampled={min(len(result), max_debug_samples)})\"\n )\n return result\n\n # [/DEF:SupersetClientGetDashboardsSummary:Function]\n\n # [DEF:SupersetClientGetDashboardsSummaryPage:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.\n # @RELATION: CALLS -> [SupersetClientGetDashboardsPage]\n def get_dashboards_summary_page(\n self,\n page: int,\n page_size: int,\n search: Optional[str] = None,\n require_slug: bool = False,\n ) -> Tuple[int, List[Dict]]:\n with belief_scope(\"SupersetClient.get_dashboards_summary_page\"):\n query: Dict[str, Any] = {\n \"page\": max(page - 1, 0),\n \"page_size\": page_size,\n }\n filters: List[Dict[str, Any]] = []\n if require_slug:\n filters.append({\"col\": \"slug\", \"opr\": \"neq\", \"value\": \"\"})\n normalized_search = (search or \"\").strip()\n if normalized_search:\n filters.append({\n \"col\": \"dashboard_title\", \"opr\": \"ct\", \"value\": normalized_search,\n })\n if filters:\n query[\"filters\"] = filters\n\n total_count, dashboards = self.get_dashboards_page(query=query)\n\n result = []\n for dash in dashboards:\n owners = self._extract_owner_labels(dash.get(\"owners\"))\n if not owners:\n owners = self._extract_owner_labels(\n [dash.get(\"created_by\"), dash.get(\"changed_by\")],\n )\n\n result.append({\n \"id\": dash.get(\"id\"),\n \"slug\": dash.get(\"slug\"),\n \"title\": dash.get(\"dashboard_title\"),\n \"url\": dash.get(\"url\"),\n \"last_modified\": dash.get(\"changed_on_utc\"),\n \"status\": \"published\" if dash.get(\"published\") else \"draft\",\n \"created_by\": self._extract_user_display(\n None, dash.get(\"created_by\"),\n ),\n \"modified_by\": self._extract_user_display(\n dash.get(\"changed_by_name\"), dash.get(\"changed_by\"),\n ),\n \"owners\": owners,\n })\n\n return total_count, result\n\n # [/DEF:SupersetClientGetDashboardsSummaryPage:Function]\n\n\n# [/DEF:SupersetDashboardsListMixin:Class]\n# [/DEF:SupersetDashboardsListMixin:Module]\n" + "body": "# [DEF:SupersetDashboardsListMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, dashboards, list, pagination, summary\n# @PURPOSE: Dashboard listing mixin for SupersetClient — paginated list, summary projection.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]\n\nimport json\nfrom typing import Any, Dict, List, Optional, Tuple, cast\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\n\nlog = MarkerLogger(\"SupersetDashboardsList\")\n\n\n# [DEF:SupersetDashboardsListMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing dashboard listing and summary projection operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetUserProjectionMixin]\nclass SupersetDashboardsListMixin:\n # [DEF:SupersetClientGetDashboards:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_dashboards\"):\n log.reason(\"Fetching dashboards\", payload={\"has_query\": query is not None})\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = [\n \"slug\", \"id\", \"url\", \"changed_on_utc\", \"dashboard_title\",\n \"published\", \"created_by\", \"changed_by\", \"changed_by_name\", \"owners\",\n ]\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dashboard/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n log.reflect(\"Dashboards fetched\", payload={\"total_count\": total_count})\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDashboards:Function]\n\n # [DEF:SupersetClientGetDashboardsPage:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.\n # @RELATION: CALLS -> [APIClient]\n def get_dashboards_page(\n self, query: Optional[Dict] = None\n ) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_dashboards_page\"):\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = [\n \"slug\", \"id\", \"url\", \"changed_on_utc\", \"dashboard_title\",\n \"published\", \"created_by\", \"changed_by\", \"changed_by_name\", \"owners\",\n ]\n\n response_json = cast(\n Dict[str, Any],\n self.network.request(\n method=\"GET\",\n endpoint=\"/dashboard/\",\n params={\"q\": json.dumps(validated_query)},\n ),\n )\n result = response_json.get(\"result\", [])\n total_count = response_json.get(\"count\", len(result))\n return total_count, result\n\n # [/DEF:SupersetClientGetDashboardsPage:Function]\n\n # [DEF:SupersetClientGetDashboardsSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches dashboard metadata optimized for the grid.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_dashboards_summary\"):\n query: Dict[str, Any] = {}\n if require_slug:\n query[\"filters\"] = [{\"col\": \"slug\", \"opr\": \"neq\", \"value\": \"\"}]\n _, dashboards = self.get_dashboards(query=query)\n\n result = []\n max_debug_samples = 12\n for index, dash in enumerate(dashboards):\n raw_owners = dash.get(\"owners\")\n raw_created_by = dash.get(\"created_by\")\n raw_changed_by = dash.get(\"changed_by\")\n raw_changed_by_name = dash.get(\"changed_by_name\")\n\n owners = self._extract_owner_labels(raw_owners)\n if not owners:\n owners = self._extract_owner_labels([raw_created_by, raw_changed_by])\n\n projected_created_by = self._extract_user_display(None, raw_created_by)\n projected_modified_by = self._extract_user_display(\n raw_changed_by_name, raw_changed_by,\n )\n\n raw_owner_usernames: List[str] = []\n if isinstance(raw_owners, list):\n for owner_payload in raw_owners:\n if isinstance(owner_payload, dict):\n owner_username = self._sanitize_user_text(\n owner_payload.get(\"username\")\n )\n if owner_username:\n raw_owner_usernames.append(owner_username)\n\n result.append({\n \"id\": dash.get(\"id\"),\n \"slug\": dash.get(\"slug\"),\n \"title\": dash.get(\"dashboard_title\"),\n \"url\": dash.get(\"url\"),\n \"last_modified\": dash.get(\"changed_on_utc\"),\n \"status\": \"published\" if dash.get(\"published\") else \"draft\",\n \"created_by\": projected_created_by,\n \"modified_by\": projected_modified_by,\n \"owners\": owners,\n })\n\n if index < max_debug_samples:\n log.reflect(\n \"Dashboard actor projection sample\",\n payload={\n \"env\": getattr(self.env, \"id\", None),\n \"dashboard_id\": dash.get(\"id\"),\n \"raw_owners\": raw_owners,\n \"raw_owner_usernames\": raw_owner_usernames,\n \"raw_created_by\": raw_created_by,\n \"raw_changed_by\": raw_changed_by,\n \"raw_changed_by_name\": raw_changed_by_name,\n \"projected_owners\": owners,\n \"projected_created_by\": projected_created_by,\n \"projected_modified_by\": projected_modified_by,\n },\n )\n\n log.reflect(\n \"Dashboard actor projection summary\",\n payload={\n \"env\": getattr(self.env, \"id\", None),\n \"dashboards\": len(result),\n \"sampled\": min(len(result), max_debug_samples),\n },\n )\n return result\n\n # [/DEF:SupersetClientGetDashboardsSummary:Function]\n\n # [DEF:SupersetClientGetDashboardsSummaryPage:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.\n # @RELATION: CALLS -> [SupersetClientGetDashboardsPage]\n def get_dashboards_summary_page(\n self,\n page: int,\n page_size: int,\n search: Optional[str] = None,\n require_slug: bool = False,\n ) -> Tuple[int, List[Dict]]:\n with belief_scope(\"SupersetClient.get_dashboards_summary_page\"):\n query: Dict[str, Any] = {\n \"page\": max(page - 1, 0),\n \"page_size\": page_size,\n }\n filters: List[Dict[str, Any]] = []\n if require_slug:\n filters.append({\"col\": \"slug\", \"opr\": \"neq\", \"value\": \"\"})\n normalized_search = (search or \"\").strip()\n if normalized_search:\n filters.append({\n \"col\": \"dashboard_title\", \"opr\": \"ct\", \"value\": normalized_search,\n })\n if filters:\n query[\"filters\"] = filters\n\n total_count, dashboards = self.get_dashboards_page(query=query)\n\n result = []\n for dash in dashboards:\n owners = self._extract_owner_labels(dash.get(\"owners\"))\n if not owners:\n owners = self._extract_owner_labels(\n [dash.get(\"created_by\"), dash.get(\"changed_by\")],\n )\n\n result.append({\n \"id\": dash.get(\"id\"),\n \"slug\": dash.get(\"slug\"),\n \"title\": dash.get(\"dashboard_title\"),\n \"url\": dash.get(\"url\"),\n \"last_modified\": dash.get(\"changed_on_utc\"),\n \"status\": \"published\" if dash.get(\"published\") else \"draft\",\n \"created_by\": self._extract_user_display(\n None, dash.get(\"created_by\"),\n ),\n \"modified_by\": self._extract_user_display(\n dash.get(\"changed_by_name\"), dash.get(\"changed_by\"),\n ),\n \"owners\": owners,\n })\n\n return total_count, result\n\n # [/DEF:SupersetClientGetDashboardsSummaryPage:Function]\n\n\n# [/DEF:SupersetDashboardsListMixin:Class]\n# [/DEF:SupersetDashboardsListMixin:Module]\n" }, { "contract_id": "SupersetClientGetDashboards", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_list.py", - "start_line": 23, - "end_line": 48, + "start_line": 24, + "end_line": 49, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39955,14 +40711,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDashboards:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_dashboards\"):\n app_logger.info(\"[get_dashboards][Enter] Fetching dashboards.\")\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = [\n \"slug\", \"id\", \"url\", \"changed_on_utc\", \"dashboard_title\",\n \"published\", \"created_by\", \"changed_by\", \"changed_by_name\", \"owners\",\n ]\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dashboard/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n app_logger.info(\"[get_dashboards][Exit] Found %d dashboards.\", total_count)\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDashboards:Function]\n" + "body": " # [DEF:SupersetClientGetDashboards:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_dashboards\"):\n log.reason(\"Fetching dashboards\", payload={\"has_query\": query is not None})\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = [\n \"slug\", \"id\", \"url\", \"changed_on_utc\", \"dashboard_title\",\n \"published\", \"created_by\", \"changed_by\", \"changed_by_name\", \"owners\",\n ]\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dashboard/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n log.reflect(\"Dashboards fetched\", payload={\"total_count\": total_count})\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDashboards:Function]\n" }, { "contract_id": "SupersetClientGetDashboardsPage", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_list.py", - "start_line": 50, - "end_line": 77, + "start_line": 51, + "end_line": 78, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -39985,8 +40741,8 @@ "contract_id": "SupersetClientGetDashboardsSummary", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_list.py", - "start_line": 79, - "end_line": 146, + "start_line": 80, + "end_line": 157, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40003,14 +40759,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDashboardsSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches dashboard metadata optimized for the grid.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_dashboards_summary\"):\n query: Dict[str, Any] = {}\n if require_slug:\n query[\"filters\"] = [{\"col\": \"slug\", \"opr\": \"neq\", \"value\": \"\"}]\n _, dashboards = self.get_dashboards(query=query)\n\n result = []\n max_debug_samples = 12\n for index, dash in enumerate(dashboards):\n raw_owners = dash.get(\"owners\")\n raw_created_by = dash.get(\"created_by\")\n raw_changed_by = dash.get(\"changed_by\")\n raw_changed_by_name = dash.get(\"changed_by_name\")\n\n owners = self._extract_owner_labels(raw_owners)\n if not owners:\n owners = self._extract_owner_labels([raw_created_by, raw_changed_by])\n\n projected_created_by = self._extract_user_display(None, raw_created_by)\n projected_modified_by = self._extract_user_display(\n raw_changed_by_name, raw_changed_by,\n )\n\n raw_owner_usernames: List[str] = []\n if isinstance(raw_owners, list):\n for owner_payload in raw_owners:\n if isinstance(owner_payload, dict):\n owner_username = self._sanitize_user_text(\n owner_payload.get(\"username\")\n )\n if owner_username:\n raw_owner_usernames.append(owner_username)\n\n result.append({\n \"id\": dash.get(\"id\"),\n \"slug\": dash.get(\"slug\"),\n \"title\": dash.get(\"dashboard_title\"),\n \"url\": dash.get(\"url\"),\n \"last_modified\": dash.get(\"changed_on_utc\"),\n \"status\": \"published\" if dash.get(\"published\") else \"draft\",\n \"created_by\": projected_created_by,\n \"modified_by\": projected_modified_by,\n \"owners\": owners,\n })\n\n if index < max_debug_samples:\n app_logger.reflect(\n \"[REFLECT] Dashboard actor projection sample \"\n f\"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, \"\n f\"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, \"\n f\"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, \"\n f\"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, \"\n f\"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})\"\n )\n\n app_logger.reflect(\n \"[REFLECT] Dashboard actor projection summary \"\n f\"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, \"\n f\"sampled={min(len(result), max_debug_samples)})\"\n )\n return result\n\n # [/DEF:SupersetClientGetDashboardsSummary:Function]\n" + "body": " # [DEF:SupersetClientGetDashboardsSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches dashboard metadata optimized for the grid.\n # @RELATION: CALLS -> [SupersetClientGetDashboards]\n def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_dashboards_summary\"):\n query: Dict[str, Any] = {}\n if require_slug:\n query[\"filters\"] = [{\"col\": \"slug\", \"opr\": \"neq\", \"value\": \"\"}]\n _, dashboards = self.get_dashboards(query=query)\n\n result = []\n max_debug_samples = 12\n for index, dash in enumerate(dashboards):\n raw_owners = dash.get(\"owners\")\n raw_created_by = dash.get(\"created_by\")\n raw_changed_by = dash.get(\"changed_by\")\n raw_changed_by_name = dash.get(\"changed_by_name\")\n\n owners = self._extract_owner_labels(raw_owners)\n if not owners:\n owners = self._extract_owner_labels([raw_created_by, raw_changed_by])\n\n projected_created_by = self._extract_user_display(None, raw_created_by)\n projected_modified_by = self._extract_user_display(\n raw_changed_by_name, raw_changed_by,\n )\n\n raw_owner_usernames: List[str] = []\n if isinstance(raw_owners, list):\n for owner_payload in raw_owners:\n if isinstance(owner_payload, dict):\n owner_username = self._sanitize_user_text(\n owner_payload.get(\"username\")\n )\n if owner_username:\n raw_owner_usernames.append(owner_username)\n\n result.append({\n \"id\": dash.get(\"id\"),\n \"slug\": dash.get(\"slug\"),\n \"title\": dash.get(\"dashboard_title\"),\n \"url\": dash.get(\"url\"),\n \"last_modified\": dash.get(\"changed_on_utc\"),\n \"status\": \"published\" if dash.get(\"published\") else \"draft\",\n \"created_by\": projected_created_by,\n \"modified_by\": projected_modified_by,\n \"owners\": owners,\n })\n\n if index < max_debug_samples:\n log.reflect(\n \"Dashboard actor projection sample\",\n payload={\n \"env\": getattr(self.env, \"id\", None),\n \"dashboard_id\": dash.get(\"id\"),\n \"raw_owners\": raw_owners,\n \"raw_owner_usernames\": raw_owner_usernames,\n \"raw_created_by\": raw_created_by,\n \"raw_changed_by\": raw_changed_by,\n \"raw_changed_by_name\": raw_changed_by_name,\n \"projected_owners\": owners,\n \"projected_created_by\": projected_created_by,\n \"projected_modified_by\": projected_modified_by,\n },\n )\n\n log.reflect(\n \"Dashboard actor projection summary\",\n payload={\n \"env\": getattr(self.env, \"id\", None),\n \"dashboards\": len(result),\n \"sampled\": min(len(result), max_debug_samples),\n },\n )\n return result\n\n # [/DEF:SupersetClientGetDashboardsSummary:Function]\n" }, { "contract_id": "SupersetClientGetDashboardsSummaryPage", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_dashboards_list.py", - "start_line": 148, - "end_line": 203, + "start_line": 159, + "end_line": 214, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40034,7 +40790,7 @@ "contract_type": "Module", "file_path": "backend/src/core/superset_client/_databases.py", "start_line": 1, - "end_line": 91, + "end_line": 92, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40060,14 +40816,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetDatabasesMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, databases, list, get, summary, uuid\n# @PURPOSE: Database domain mixin for SupersetClient — list, get, summary, by_uuid.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n\nfrom typing import Any, Dict, List, Optional, Tuple, cast\n\nfrom ..logger import logger as app_logger, belief_scope\n\napp_logger = cast(Any, app_logger)\n\n\n# [DEF:SupersetDatabasesMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing all database-related Superset API operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\nclass SupersetDatabasesMixin:\n # [DEF:SupersetClientGetDatabases:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список баз данных.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_databases\"):\n app_logger.info(\"[get_databases][Enter] Fetching databases.\")\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = []\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/database/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n app_logger.info(\"[get_databases][Exit] Found %d databases.\", total_count)\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatabases:Function]\n\n # [DEF:SupersetClientGetDatabase:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретной базе данных по её ID.\n # @RELATION: CALLS -> [APIClient]\n def get_database(self, database_id: int) -> Dict:\n with belief_scope(\"get_database\"):\n app_logger.info(\"[get_database][Enter] Fetching database %s.\", database_id)\n response = self.network.request(\n method=\"GET\", endpoint=f\"/database/{database_id}\"\n )\n response = cast(Dict, response)\n app_logger.info(\"[get_database][Exit] Got database %s.\", database_id)\n return response\n\n # [/DEF:SupersetClientGetDatabase:Function]\n\n # [DEF:SupersetClientGetDatabasesSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch a summary of databases including uuid, name, and engine.\n # @RELATION: CALLS -> [SupersetClientGetDatabases]\n def get_databases_summary(self) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_databases_summary\"):\n query = {\"columns\": [\"id\", \"uuid\", \"database_name\", \"backend\"]}\n _, databases = self.get_databases(query=query)\n\n # Map 'backend' to 'engine' for consistency with contracts\n for db in databases:\n db[\"engine\"] = db.pop(\"backend\", None)\n\n return databases\n\n # [/DEF:SupersetClientGetDatabasesSummary:Function]\n\n # [DEF:SupersetClientGetDatabaseByUuid:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Find a database by its UUID.\n # @RELATION: CALLS -> [SupersetClientGetDatabases]\n def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:\n with belief_scope(\"SupersetClient.get_database_by_uuid\", f\"uuid={db_uuid}\"):\n query = {\"filters\": [{\"col\": \"uuid\", \"op\": \"eq\", \"value\": db_uuid}]}\n _, databases = self.get_databases(query=query)\n return databases[0] if databases else None\n\n # [/DEF:SupersetClientGetDatabaseByUuid:Function]\n\n\n# [/DEF:SupersetDatabasesMixin:Class]\n# [/DEF:SupersetDatabasesMixin:Module]\n" + "body": "# [DEF:SupersetDatabasesMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, databases, list, get, summary, uuid\n# @PURPOSE: Database domain mixin for SupersetClient — list, get, summary, by_uuid.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n\nfrom typing import Any, Dict, List, Optional, Tuple, cast\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\n\nlog = MarkerLogger(\"SupersetDatabases\")\n\n\n# [DEF:SupersetDatabasesMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing all database-related Superset API operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\nclass SupersetDatabasesMixin:\n # [DEF:SupersetClientGetDatabases:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список баз данных.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_databases\"):\n log.reason(\"Fetching databases\", payload={\"has_query\": query is not None})\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = []\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/database/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n log.reflect(\"Databases fetched\", payload={\"total_count\": total_count})\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatabases:Function]\n\n # [DEF:SupersetClientGetDatabase:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретной базе данных по её ID.\n # @RELATION: CALLS -> [APIClient]\n def get_database(self, database_id: int) -> Dict:\n with belief_scope(\"get_database\"):\n log.reason(\"Fetching database\", payload={\"database_id\": database_id})\n response = self.network.request(\n method=\"GET\", endpoint=f\"/database/{database_id}\"\n )\n response = cast(Dict, response)\n log.reflect(\"Database fetched\", payload={\"database_id\": database_id})\n return response\n\n # [/DEF:SupersetClientGetDatabase:Function]\n\n # [DEF:SupersetClientGetDatabasesSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch a summary of databases including uuid, name, and engine.\n # @RELATION: CALLS -> [SupersetClientGetDatabases]\n def get_databases_summary(self) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_databases_summary\"):\n query = {\"columns\": [\"id\", \"uuid\", \"database_name\", \"backend\"]}\n _, databases = self.get_databases(query=query)\n\n # Map 'backend' to 'engine' for consistency with contracts\n for db in databases:\n db[\"engine\"] = db.pop(\"backend\", None)\n\n return databases\n\n # [/DEF:SupersetClientGetDatabasesSummary:Function]\n\n # [DEF:SupersetClientGetDatabaseByUuid:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Find a database by its UUID.\n # @RELATION: CALLS -> [SupersetClientGetDatabases]\n def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:\n with belief_scope(\"SupersetClient.get_database_by_uuid\", f\"uuid={db_uuid}\"):\n query = {\"filters\": [{\"col\": \"uuid\", \"op\": \"eq\", \"value\": db_uuid}]}\n _, databases = self.get_databases(query=query)\n return databases[0] if databases else None\n\n # [/DEF:SupersetClientGetDatabaseByUuid:Function]\n\n\n# [/DEF:SupersetDatabasesMixin:Class]\n# [/DEF:SupersetDatabasesMixin:Module]\n" }, { "contract_id": "SupersetClientGetDatabases", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_databases.py", - "start_line": 20, - "end_line": 42, + "start_line": 21, + "end_line": 43, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40084,14 +40840,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDatabases:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список баз данных.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_databases\"):\n app_logger.info(\"[get_databases][Enter] Fetching databases.\")\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = []\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/database/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n app_logger.info(\"[get_databases][Exit] Found %d databases.\", total_count)\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatabases:Function]\n" + "body": " # [DEF:SupersetClientGetDatabases:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список баз данных.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_databases\"):\n log.reason(\"Fetching databases\", payload={\"has_query\": query is not None})\n validated_query = self._validate_query_params(query or {})\n if \"columns\" not in validated_query:\n validated_query[\"columns\"] = []\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/database/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n log.reflect(\"Databases fetched\", payload={\"total_count\": total_count})\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatabases:Function]\n" }, { "contract_id": "SupersetClientGetDatabase", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_databases.py", - "start_line": 44, - "end_line": 58, + "start_line": 45, + "end_line": 59, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40108,14 +40864,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDatabase:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретной базе данных по её ID.\n # @RELATION: CALLS -> [APIClient]\n def get_database(self, database_id: int) -> Dict:\n with belief_scope(\"get_database\"):\n app_logger.info(\"[get_database][Enter] Fetching database %s.\", database_id)\n response = self.network.request(\n method=\"GET\", endpoint=f\"/database/{database_id}\"\n )\n response = cast(Dict, response)\n app_logger.info(\"[get_database][Exit] Got database %s.\", database_id)\n return response\n\n # [/DEF:SupersetClientGetDatabase:Function]\n" + "body": " # [DEF:SupersetClientGetDatabase:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретной базе данных по её ID.\n # @RELATION: CALLS -> [APIClient]\n def get_database(self, database_id: int) -> Dict:\n with belief_scope(\"get_database\"):\n log.reason(\"Fetching database\", payload={\"database_id\": database_id})\n response = self.network.request(\n method=\"GET\", endpoint=f\"/database/{database_id}\"\n )\n response = cast(Dict, response)\n log.reflect(\"Database fetched\", payload={\"database_id\": database_id})\n return response\n\n # [/DEF:SupersetClientGetDatabase:Function]\n" }, { "contract_id": "SupersetClientGetDatabasesSummary", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_databases.py", - "start_line": 60, - "end_line": 75, + "start_line": 61, + "end_line": 76, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40138,8 +40894,8 @@ "contract_id": "SupersetClientGetDatabaseByUuid", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_databases.py", - "start_line": 77, - "end_line": 87, + "start_line": 78, + "end_line": 88, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40163,7 +40919,7 @@ "contract_type": "Module", "file_path": "backend/src/core/superset_client/_datasets.py", "start_line": 1, - "end_line": 219, + "end_line": 223, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40189,14 +40945,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetDatasetsMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, datasets, list, get, detail, update\n# @PURPOSE: Dataset domain mixin for SupersetClient — list, get, detail, update.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n\nimport json\nfrom typing import Any, Dict, List, Optional, Tuple, cast\n\nfrom ..logger import logger as app_logger, belief_scope\n\napp_logger = cast(Any, app_logger)\n\n\n# [DEF:SupersetDatasetsMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing basic dataset CRUD operations (list, get, detail, update).\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\nclass SupersetDatasetsMixin:\n # [DEF:SupersetClientGetDatasets:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_datasets\"):\n app_logger.info(\"[get_datasets][Enter] Fetching datasets.\")\n validated_query = self._validate_query_params(query)\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dataset/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n app_logger.info(\"[get_datasets][Exit] Found %d datasets.\", total_count)\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatasets:Function]\n\n # [DEF:SupersetClientGetDatasetsSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.\n # @RELATION: CALLS -> [SupersetClientGetDatasets]\n def get_datasets_summary(self) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_datasets_summary\"):\n query = {\"columns\": [\"id\", \"table_name\", \"schema\", \"database\"]}\n _, datasets = self.get_datasets(query=query)\n\n result = []\n for ds in datasets:\n result.append(\n {\n \"id\": ds.get(\"id\"),\n \"table_name\": ds.get(\"table_name\"),\n \"schema\": ds.get(\"schema\"),\n \"database\": ds.get(\"database\", {}).get(\n \"database_name\", \"Unknown\"\n ),\n }\n )\n return result\n\n # [/DEF:SupersetClientGetDatasetsSummary:Function]\n\n # [DEF:SupersetClientGetDatasetDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards.\n # @RELATION: CALLS -> [SupersetClientGetDataset]\n # @RELATION: CALLS -> [APIClient]\n def get_dataset_detail(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset_detail\", f\"id={dataset_id}\"):\n\n def as_bool(value, default=False):\n if value is None:\n return default\n if isinstance(value, bool):\n return value\n if isinstance(value, str):\n return value.strip().lower() in (\"1\", \"true\", \"yes\", \"y\", \"on\")\n return bool(value)\n\n response = self.get_dataset(dataset_id)\n\n if isinstance(response, dict) and \"result\" in response:\n dataset = response[\"result\"]\n else:\n dataset = response\n\n columns = dataset.get(\"columns\", [])\n column_info = []\n for col in columns:\n col_id = col.get(\"id\")\n if col_id is None:\n continue\n column_info.append(\n {\n \"id\": int(col_id),\n \"name\": col.get(\"column_name\"),\n \"type\": col.get(\"type\"),\n \"is_dttm\": as_bool(col.get(\"is_dttm\"), default=False),\n \"is_active\": as_bool(col.get(\"is_active\"), default=True),\n \"description\": col.get(\"description\", \"\"),\n }\n )\n\n linked_dashboards = []\n try:\n related_objects = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}/related_objects\"\n )\n\n if isinstance(related_objects, dict):\n if \"dashboards\" in related_objects:\n dashboards_data = related_objects[\"dashboards\"]\n elif \"result\" in related_objects and isinstance(\n related_objects[\"result\"], dict\n ):\n dashboards_data = related_objects[\"result\"].get(\"dashboards\", [])\n else:\n dashboards_data = []\n\n for dash in dashboards_data:\n if isinstance(dash, dict):\n dash_id = dash.get(\"id\")\n if dash_id is None:\n continue\n linked_dashboards.append(\n {\n \"id\": int(dash_id),\n \"title\": dash.get(\"dashboard_title\")\n or dash.get(\"title\", f\"Dashboard {dash_id}\"),\n \"slug\": dash.get(\"slug\"),\n }\n )\n else:\n try:\n dash_id = int(dash)\n except (TypeError, ValueError):\n continue\n linked_dashboards.append(\n {\"id\": dash_id, \"title\": f\"Dashboard {dash_id}\", \"slug\": None}\n )\n except Exception as e:\n app_logger.warning(\n f\"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}\"\n )\n linked_dashboards = []\n\n database_obj = dataset.get(\"database\")\n database_dict = database_obj if isinstance(database_obj, dict) else {}\n sql = dataset.get(\"sql\", \"\")\n\n result = {\n \"id\": dataset.get(\"id\"),\n \"table_name\": dataset.get(\"table_name\"),\n \"schema\": dataset.get(\"schema\"),\n \"database\": database_dict,\n \"database_id\": dataset.get(\"database_id\", database_dict.get(\"id\")),\n \"database_backend\": database_dict.get(\"backend\") or database_dict.get(\"engine\") or \"\",\n \"description\": dataset.get(\"description\", \"\"),\n \"columns\": column_info,\n \"column_count\": len(column_info),\n \"sql\": sql,\n \"linked_dashboards\": linked_dashboards,\n \"linked_dashboard_count\": len(linked_dashboards),\n \"is_sqllab_view\": as_bool(dataset.get(\"is_sqllab_view\"), default=False),\n \"created_on\": dataset.get(\"created_on\"),\n \"changed_on\": dataset.get(\"changed_on\"),\n }\n\n app_logger.info(\n f\"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards\"\n )\n return result\n\n # [/DEF:SupersetClientGetDatasetDetail:Function]\n\n # [DEF:SupersetClientGetDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретном датасете по его ID.\n # @RELATION: CALLS -> [APIClient]\n def get_dataset(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset\", f\"id={dataset_id}\"):\n app_logger.info(\"[get_dataset][Enter] Fetching dataset %s.\", dataset_id)\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}\"\n )\n response = cast(Dict, response)\n app_logger.info(\"[get_dataset][Exit] Got dataset %s.\", dataset_id)\n return response\n\n # [/DEF:SupersetClientGetDataset:Function]\n\n # [DEF:SupersetClientUpdateDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Обновляет данные датасета по его ID.\n # @SIDE_EFFECT: Modifies resource in upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def update_dataset(self, dataset_id: int, data: Dict) -> Dict:\n with belief_scope(\"SupersetClient.update_dataset\", f\"id={dataset_id}\"):\n app_logger.info(\"[update_dataset][Enter] Updating dataset %s.\", dataset_id)\n response = self.network.request(\n method=\"PUT\",\n endpoint=f\"/dataset/{dataset_id}\",\n data=json.dumps(data),\n headers={\"Content-Type\": \"application/json\"},\n )\n response = cast(Dict, response)\n app_logger.info(\"[update_dataset][Exit] Updated dataset %s.\", dataset_id)\n return response\n\n # [/DEF:SupersetClientUpdateDataset:Function]\n\n\n# [/DEF:SupersetDatasetsMixin:Class]\n# [/DEF:SupersetDatasetsMixin:Module]\n" + "body": "# [DEF:SupersetDatasetsMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, datasets, list, get, detail, update\n# @PURPOSE: Dataset domain mixin for SupersetClient — list, get, detail, update.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n\nimport json\nfrom typing import Any, Dict, List, Optional, Tuple, cast\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\n\nlog = MarkerLogger(\"SupersetDatasets\")\n\n\n# [DEF:SupersetDatasetsMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing basic dataset CRUD operations (list, get, detail, update).\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\nclass SupersetDatasetsMixin:\n # [DEF:SupersetClientGetDatasets:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_datasets\"):\n log.reason(\"Fetching datasets\")\n validated_query = self._validate_query_params(query)\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dataset/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n log.reflect(\"Datasets fetched\", payload={\"total_count\": total_count})\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatasets:Function]\n\n # [DEF:SupersetClientGetDatasetsSummary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.\n # @RELATION: CALLS -> [SupersetClientGetDatasets]\n def get_datasets_summary(self) -> List[Dict]:\n with belief_scope(\"SupersetClient.get_datasets_summary\"):\n query = {\"columns\": [\"id\", \"table_name\", \"schema\", \"database\"]}\n _, datasets = self.get_datasets(query=query)\n\n result = []\n for ds in datasets:\n result.append(\n {\n \"id\": ds.get(\"id\"),\n \"table_name\": ds.get(\"table_name\"),\n \"schema\": ds.get(\"schema\"),\n \"database\": ds.get(\"database\", {}).get(\n \"database_name\", \"Unknown\"\n ),\n }\n )\n return result\n\n # [/DEF:SupersetClientGetDatasetsSummary:Function]\n\n # [DEF:SupersetClientGetDatasetDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards.\n # @RELATION: CALLS -> [SupersetClientGetDataset]\n # @RELATION: CALLS -> [APIClient]\n def get_dataset_detail(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset_detail\", f\"id={dataset_id}\"):\n\n def as_bool(value, default=False):\n if value is None:\n return default\n if isinstance(value, bool):\n return value\n if isinstance(value, str):\n return value.strip().lower() in (\"1\", \"true\", \"yes\", \"y\", \"on\")\n return bool(value)\n\n response = self.get_dataset(dataset_id)\n\n if isinstance(response, dict) and \"result\" in response:\n dataset = response[\"result\"]\n else:\n dataset = response\n\n columns = dataset.get(\"columns\", [])\n column_info = []\n for col in columns:\n col_id = col.get(\"id\")\n if col_id is None:\n continue\n column_info.append(\n {\n \"id\": int(col_id),\n \"name\": col.get(\"column_name\"),\n \"type\": col.get(\"type\"),\n \"is_dttm\": as_bool(col.get(\"is_dttm\"), default=False),\n \"is_active\": as_bool(col.get(\"is_active\"), default=True),\n \"description\": col.get(\"description\", \"\"),\n }\n )\n\n linked_dashboards = []\n try:\n related_objects = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}/related_objects\"\n )\n\n if isinstance(related_objects, dict):\n if \"dashboards\" in related_objects:\n dashboards_data = related_objects[\"dashboards\"]\n elif \"result\" in related_objects and isinstance(\n related_objects[\"result\"], dict\n ):\n dashboards_data = related_objects[\"result\"].get(\"dashboards\", [])\n else:\n dashboards_data = []\n\n for dash in dashboards_data:\n if isinstance(dash, dict):\n dash_id = dash.get(\"id\")\n if dash_id is None:\n continue\n linked_dashboards.append(\n {\n \"id\": int(dash_id),\n \"title\": dash.get(\"dashboard_title\")\n or dash.get(\"title\", f\"Dashboard {dash_id}\"),\n \"slug\": dash.get(\"slug\"),\n }\n )\n else:\n try:\n dash_id = int(dash)\n except (TypeError, ValueError):\n continue\n linked_dashboards.append(\n {\"id\": dash_id, \"title\": f\"Dashboard {dash_id}\", \"slug\": None}\n )\n except Exception as e:\n log.explore(\"Failed to fetch related dashboards\", error=str(e), payload={\"dataset_id\": dataset_id})\n linked_dashboards = []\n\n database_obj = dataset.get(\"database\")\n database_dict = database_obj if isinstance(database_obj, dict) else {}\n sql = dataset.get(\"sql\", \"\")\n\n result = {\n \"id\": dataset.get(\"id\"),\n \"table_name\": dataset.get(\"table_name\"),\n \"schema\": dataset.get(\"schema\"),\n \"database\": database_dict,\n \"database_id\": dataset.get(\"database_id\", database_dict.get(\"id\")),\n \"database_backend\": database_dict.get(\"backend\") or database_dict.get(\"engine\") or \"\",\n \"description\": dataset.get(\"description\", \"\"),\n \"columns\": column_info,\n \"column_count\": len(column_info),\n \"sql\": sql,\n \"linked_dashboards\": linked_dashboards,\n \"linked_dashboard_count\": len(linked_dashboards),\n \"is_sqllab_view\": as_bool(dataset.get(\"is_sqllab_view\"), default=False),\n \"created_on\": dataset.get(\"created_on\"),\n \"changed_on\": dataset.get(\"changed_on\"),\n }\n\n log.reflect(\n \"Dataset detail fetched\",\n payload={\n \"dataset_id\": dataset_id,\n \"column_count\": len(column_info),\n \"linked_dashboard_count\": len(linked_dashboards),\n },\n )\n return result\n\n # [/DEF:SupersetClientGetDatasetDetail:Function]\n\n # [DEF:SupersetClientGetDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретном датасете по его ID.\n # @RELATION: CALLS -> [APIClient]\n def get_dataset(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset\", f\"id={dataset_id}\"):\n log.reason(\"Fetching dataset\", payload={\"dataset_id\": dataset_id})\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}\"\n )\n response = cast(Dict, response)\n log.reflect(\"Dataset fetched\", payload={\"dataset_id\": dataset_id})\n return response\n\n # [/DEF:SupersetClientGetDataset:Function]\n\n # [DEF:SupersetClientUpdateDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Обновляет данные датасета по его ID.\n # @SIDE_EFFECT: Modifies resource in upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def update_dataset(self, dataset_id: int, data: Dict) -> Dict:\n with belief_scope(\"SupersetClient.update_dataset\", f\"id={dataset_id}\"):\n log.reason(\"Updating dataset\", payload={\"dataset_id\": dataset_id})\n response = self.network.request(\n method=\"PUT\",\n endpoint=f\"/dataset/{dataset_id}\",\n data=json.dumps(data),\n headers={\"Content-Type\": \"application/json\"},\n )\n response = cast(Dict, response)\n log.reflect(\"Dataset updated\", payload={\"dataset_id\": dataset_id})\n return response\n\n # [/DEF:SupersetClientUpdateDataset:Function]\n\n\n# [/DEF:SupersetDatasetsMixin:Class]\n# [/DEF:SupersetDatasetsMixin:Module]\n" }, { "contract_id": "SupersetClientGetDatasets", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets.py", - "start_line": 21, - "end_line": 41, + "start_line": 22, + "end_line": 42, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40213,14 +40969,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDatasets:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_datasets\"):\n app_logger.info(\"[get_datasets][Enter] Fetching datasets.\")\n validated_query = self._validate_query_params(query)\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dataset/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n app_logger.info(\"[get_datasets][Exit] Found %d datasets.\", total_count)\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatasets:Function]\n" + "body": " # [DEF:SupersetClientGetDatasets:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.\n # @RELATION: CALLS -> [SupersetClientFetchAllPages]\n def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:\n with belief_scope(\"get_datasets\"):\n log.reason(\"Fetching datasets\")\n validated_query = self._validate_query_params(query)\n\n paginated_data = self._fetch_all_pages(\n endpoint=\"/dataset/\",\n pagination_options={\n \"base_query\": validated_query,\n \"results_field\": \"result\",\n },\n )\n total_count = len(paginated_data)\n log.reflect(\"Datasets fetched\", payload={\"total_count\": total_count})\n return total_count, paginated_data\n\n # [/DEF:SupersetClientGetDatasets:Function]\n" }, { "contract_id": "SupersetClientGetDatasetsSummary", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets.py", - "start_line": 43, - "end_line": 66, + "start_line": 44, + "end_line": 67, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40243,8 +40999,8 @@ "contract_id": "SupersetClientGetDatasetDetail", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets.py", - "start_line": 68, - "end_line": 179, + "start_line": 69, + "end_line": 183, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40267,14 +41023,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDatasetDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards.\n # @RELATION: CALLS -> [SupersetClientGetDataset]\n # @RELATION: CALLS -> [APIClient]\n def get_dataset_detail(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset_detail\", f\"id={dataset_id}\"):\n\n def as_bool(value, default=False):\n if value is None:\n return default\n if isinstance(value, bool):\n return value\n if isinstance(value, str):\n return value.strip().lower() in (\"1\", \"true\", \"yes\", \"y\", \"on\")\n return bool(value)\n\n response = self.get_dataset(dataset_id)\n\n if isinstance(response, dict) and \"result\" in response:\n dataset = response[\"result\"]\n else:\n dataset = response\n\n columns = dataset.get(\"columns\", [])\n column_info = []\n for col in columns:\n col_id = col.get(\"id\")\n if col_id is None:\n continue\n column_info.append(\n {\n \"id\": int(col_id),\n \"name\": col.get(\"column_name\"),\n \"type\": col.get(\"type\"),\n \"is_dttm\": as_bool(col.get(\"is_dttm\"), default=False),\n \"is_active\": as_bool(col.get(\"is_active\"), default=True),\n \"description\": col.get(\"description\", \"\"),\n }\n )\n\n linked_dashboards = []\n try:\n related_objects = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}/related_objects\"\n )\n\n if isinstance(related_objects, dict):\n if \"dashboards\" in related_objects:\n dashboards_data = related_objects[\"dashboards\"]\n elif \"result\" in related_objects and isinstance(\n related_objects[\"result\"], dict\n ):\n dashboards_data = related_objects[\"result\"].get(\"dashboards\", [])\n else:\n dashboards_data = []\n\n for dash in dashboards_data:\n if isinstance(dash, dict):\n dash_id = dash.get(\"id\")\n if dash_id is None:\n continue\n linked_dashboards.append(\n {\n \"id\": int(dash_id),\n \"title\": dash.get(\"dashboard_title\")\n or dash.get(\"title\", f\"Dashboard {dash_id}\"),\n \"slug\": dash.get(\"slug\"),\n }\n )\n else:\n try:\n dash_id = int(dash)\n except (TypeError, ValueError):\n continue\n linked_dashboards.append(\n {\"id\": dash_id, \"title\": f\"Dashboard {dash_id}\", \"slug\": None}\n )\n except Exception as e:\n app_logger.warning(\n f\"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}\"\n )\n linked_dashboards = []\n\n database_obj = dataset.get(\"database\")\n database_dict = database_obj if isinstance(database_obj, dict) else {}\n sql = dataset.get(\"sql\", \"\")\n\n result = {\n \"id\": dataset.get(\"id\"),\n \"table_name\": dataset.get(\"table_name\"),\n \"schema\": dataset.get(\"schema\"),\n \"database\": database_dict,\n \"database_id\": dataset.get(\"database_id\", database_dict.get(\"id\")),\n \"database_backend\": database_dict.get(\"backend\") or database_dict.get(\"engine\") or \"\",\n \"description\": dataset.get(\"description\", \"\"),\n \"columns\": column_info,\n \"column_count\": len(column_info),\n \"sql\": sql,\n \"linked_dashboards\": linked_dashboards,\n \"linked_dashboard_count\": len(linked_dashboards),\n \"is_sqllab_view\": as_bool(dataset.get(\"is_sqllab_view\"), default=False),\n \"created_on\": dataset.get(\"created_on\"),\n \"changed_on\": dataset.get(\"changed_on\"),\n }\n\n app_logger.info(\n f\"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards\"\n )\n return result\n\n # [/DEF:SupersetClientGetDatasetDetail:Function]\n" + "body": " # [DEF:SupersetClientGetDatasetDetail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards.\n # @RELATION: CALLS -> [SupersetClientGetDataset]\n # @RELATION: CALLS -> [APIClient]\n def get_dataset_detail(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset_detail\", f\"id={dataset_id}\"):\n\n def as_bool(value, default=False):\n if value is None:\n return default\n if isinstance(value, bool):\n return value\n if isinstance(value, str):\n return value.strip().lower() in (\"1\", \"true\", \"yes\", \"y\", \"on\")\n return bool(value)\n\n response = self.get_dataset(dataset_id)\n\n if isinstance(response, dict) and \"result\" in response:\n dataset = response[\"result\"]\n else:\n dataset = response\n\n columns = dataset.get(\"columns\", [])\n column_info = []\n for col in columns:\n col_id = col.get(\"id\")\n if col_id is None:\n continue\n column_info.append(\n {\n \"id\": int(col_id),\n \"name\": col.get(\"column_name\"),\n \"type\": col.get(\"type\"),\n \"is_dttm\": as_bool(col.get(\"is_dttm\"), default=False),\n \"is_active\": as_bool(col.get(\"is_active\"), default=True),\n \"description\": col.get(\"description\", \"\"),\n }\n )\n\n linked_dashboards = []\n try:\n related_objects = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}/related_objects\"\n )\n\n if isinstance(related_objects, dict):\n if \"dashboards\" in related_objects:\n dashboards_data = related_objects[\"dashboards\"]\n elif \"result\" in related_objects and isinstance(\n related_objects[\"result\"], dict\n ):\n dashboards_data = related_objects[\"result\"].get(\"dashboards\", [])\n else:\n dashboards_data = []\n\n for dash in dashboards_data:\n if isinstance(dash, dict):\n dash_id = dash.get(\"id\")\n if dash_id is None:\n continue\n linked_dashboards.append(\n {\n \"id\": int(dash_id),\n \"title\": dash.get(\"dashboard_title\")\n or dash.get(\"title\", f\"Dashboard {dash_id}\"),\n \"slug\": dash.get(\"slug\"),\n }\n )\n else:\n try:\n dash_id = int(dash)\n except (TypeError, ValueError):\n continue\n linked_dashboards.append(\n {\"id\": dash_id, \"title\": f\"Dashboard {dash_id}\", \"slug\": None}\n )\n except Exception as e:\n log.explore(\"Failed to fetch related dashboards\", error=str(e), payload={\"dataset_id\": dataset_id})\n linked_dashboards = []\n\n database_obj = dataset.get(\"database\")\n database_dict = database_obj if isinstance(database_obj, dict) else {}\n sql = dataset.get(\"sql\", \"\")\n\n result = {\n \"id\": dataset.get(\"id\"),\n \"table_name\": dataset.get(\"table_name\"),\n \"schema\": dataset.get(\"schema\"),\n \"database\": database_dict,\n \"database_id\": dataset.get(\"database_id\", database_dict.get(\"id\")),\n \"database_backend\": database_dict.get(\"backend\") or database_dict.get(\"engine\") or \"\",\n \"description\": dataset.get(\"description\", \"\"),\n \"columns\": column_info,\n \"column_count\": len(column_info),\n \"sql\": sql,\n \"linked_dashboards\": linked_dashboards,\n \"linked_dashboard_count\": len(linked_dashboards),\n \"is_sqllab_view\": as_bool(dataset.get(\"is_sqllab_view\"), default=False),\n \"created_on\": dataset.get(\"created_on\"),\n \"changed_on\": dataset.get(\"changed_on\"),\n }\n\n log.reflect(\n \"Dataset detail fetched\",\n payload={\n \"dataset_id\": dataset_id,\n \"column_count\": len(column_info),\n \"linked_dashboard_count\": len(linked_dashboards),\n },\n )\n return result\n\n # [/DEF:SupersetClientGetDatasetDetail:Function]\n" }, { "contract_id": "SupersetClientGetDataset", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets.py", - "start_line": 181, - "end_line": 195, + "start_line": 185, + "end_line": 199, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40291,14 +41047,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientGetDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретном датасете по его ID.\n # @RELATION: CALLS -> [APIClient]\n def get_dataset(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset\", f\"id={dataset_id}\"):\n app_logger.info(\"[get_dataset][Enter] Fetching dataset %s.\", dataset_id)\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}\"\n )\n response = cast(Dict, response)\n app_logger.info(\"[get_dataset][Exit] Got dataset %s.\", dataset_id)\n return response\n\n # [/DEF:SupersetClientGetDataset:Function]\n" + "body": " # [DEF:SupersetClientGetDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Получает информацию о конкретном датасете по его ID.\n # @RELATION: CALLS -> [APIClient]\n def get_dataset(self, dataset_id: int) -> Dict:\n with belief_scope(\"SupersetClient.get_dataset\", f\"id={dataset_id}\"):\n log.reason(\"Fetching dataset\", payload={\"dataset_id\": dataset_id})\n response = self.network.request(\n method=\"GET\", endpoint=f\"/dataset/{dataset_id}\"\n )\n response = cast(Dict, response)\n log.reflect(\"Dataset fetched\", payload={\"dataset_id\": dataset_id})\n return response\n\n # [/DEF:SupersetClientGetDataset:Function]\n" }, { "contract_id": "SupersetClientUpdateDataset", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets.py", - "start_line": 197, - "end_line": 215, + "start_line": 201, + "end_line": 219, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40326,7 +41082,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientUpdateDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Обновляет данные датасета по его ID.\n # @SIDE_EFFECT: Modifies resource in upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def update_dataset(self, dataset_id: int, data: Dict) -> Dict:\n with belief_scope(\"SupersetClient.update_dataset\", f\"id={dataset_id}\"):\n app_logger.info(\"[update_dataset][Enter] Updating dataset %s.\", dataset_id)\n response = self.network.request(\n method=\"PUT\",\n endpoint=f\"/dataset/{dataset_id}\",\n data=json.dumps(data),\n headers={\"Content-Type\": \"application/json\"},\n )\n response = cast(Dict, response)\n app_logger.info(\"[update_dataset][Exit] Updated dataset %s.\", dataset_id)\n return response\n\n # [/DEF:SupersetClientUpdateDataset:Function]\n" + "body": " # [DEF:SupersetClientUpdateDataset:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Обновляет данные датасета по его ID.\n # @SIDE_EFFECT: Modifies resource in upstream Superset environment.\n # @RELATION: CALLS -> [APIClient]\n def update_dataset(self, dataset_id: int, data: Dict) -> Dict:\n with belief_scope(\"SupersetClient.update_dataset\", f\"id={dataset_id}\"):\n log.reason(\"Updating dataset\", payload={\"dataset_id\": dataset_id})\n response = self.network.request(\n method=\"PUT\",\n endpoint=f\"/dataset/{dataset_id}\",\n data=json.dumps(data),\n headers={\"Content-Type\": \"application/json\"},\n )\n response = cast(Dict, response)\n log.reflect(\"Dataset updated\", payload={\"dataset_id\": dataset_id})\n return response\n\n # [/DEF:SupersetClientUpdateDataset:Function]\n" }, { "contract_id": "SupersetDatasetsPreviewMixin", @@ -40393,14 +41149,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SupersetDatasetsPreviewMixin:Module]\n# @COMPLEXITY: 4\n# @LAYER: Infra\n# @SEMANTICS: superset, datasets, preview, sql, compilation, filters\n# @PURPOSE: Dataset preview compilation mixin for SupersetClient — build query context, compile SQL.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDatasetsMixin]\n\nimport json\nfrom copy import deepcopy\nfrom typing import Any, Dict, List, Optional\n\nfrom ..logger import logger as app_logger, belief_scope\nfrom ..utils.network import SupersetAPIError\n# [DEF:SupersetDatasetsPreviewMixin:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Mixin providing dataset preview compilation and query context building.\nclass SupersetDatasetsPreviewMixin:\n # [DEF:SupersetClientCompileDatasetPreview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.\n def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:\n with belief_scope('SupersetClientCompileDatasetPreview'):\n app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview')\n dataset_response = self.get_dataset(dataset_id)\n dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {}\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str)\n request_payload = json.dumps(query_context)\n strategy_attempts: List[Dict[str, Any]] = []\n strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}]\n for candidate in strategy_candidates:\n endpoint_kind = candidate['endpoint_kind']\n endpoint_path = candidate['endpoint']\n request_transport = candidate['request_transport']\n request_params = deepcopy(candidate.get('params') or {})\n request_body = candidate.get('data')\n request_headers = deepcopy(candidate.get('headers') or {})\n request_param_keys = sorted(request_params.keys())\n request_payload_keys: List[str] = []\n if isinstance(request_body, str):\n try:\n decoded_request_body = json.loads(request_body)\n if isinstance(decoded_request_body, dict):\n request_payload_keys = sorted(decoded_request_body.keys())\n except json.JSONDecodeError:\n request_payload_keys = []\n elif isinstance(request_body, dict):\n request_payload_keys = sorted(request_body.keys())\n strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys}\n app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})\n try:\n response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None)\n normalized = self._extract_compiled_sql_from_preview_response(response)\n normalized['query_context'] = query_context\n normalized['legacy_form_data'] = legacy_form_data\n normalized['endpoint'] = endpoint_path\n normalized['endpoint_kind'] = endpoint_kind\n normalized['dataset_id'] = dataset_id\n normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}]\n app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})\n app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview')\n return normalized\n except Exception as exc:\n failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)}\n strategy_attempts.append(failure_diagnostics)\n app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body})\n raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})')\n # [/DEF:SupersetClientCompileDatasetPreview:Function]\n\n # [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.\n def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'):\n app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters)\n query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {})\n legacy_form_data = deepcopy(query_context.get('form_data', {}))\n legacy_form_data.pop('datasource', None)\n legacy_form_data['metrics'] = deepcopy(query_object.get('metrics', ['count']))\n legacy_form_data['columns'] = deepcopy(query_object.get('columns', []))\n legacy_form_data['orderby'] = deepcopy(query_object.get('orderby', []))\n legacy_form_data['annotation_layers'] = deepcopy(query_object.get('annotation_layers', []))\n legacy_form_data['row_limit'] = query_object.get('row_limit', 1000)\n legacy_form_data['series_limit'] = query_object.get('series_limit', 0)\n legacy_form_data['url_params'] = deepcopy(query_object.get('url_params', template_params))\n legacy_form_data['applied_time_extras'] = deepcopy(query_object.get('applied_time_extras', {}))\n legacy_form_data['result_format'] = query_context.get('result_format', 'json')\n legacy_form_data['result_type'] = query_context.get('result_type', 'query')\n legacy_form_data['force'] = bool(query_context.get('force', True))\n extras = query_object.get('extras')\n if isinstance(extras, dict):\n legacy_form_data['extras'] = deepcopy(extras)\n time_range = query_object.get('time_range')\n if time_range:\n legacy_form_data['time_range'] = time_range\n app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})\n app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n return legacy_form_data\n # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n\n # [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.\n def build_dataset_preview_query_context(\n self,\n dataset_id: int,\n dataset_record: Dict[str, Any],\n template_params: Dict[str, Any],\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetClientBuildDatasetPreviewQueryContext\"):\n app_logger.reason(\n \"Building Superset dataset preview query context\",\n extra={\"dataset_id\": dataset_id, \"filter_count\": len(effective_filters or [])},\n )\n normalized_template_params = deepcopy(template_params or {})\n normalized_filter_payload = (\n self._normalize_effective_filters_for_query_context(\n effective_filters or []\n )\n )\n normalized_filters = normalized_filter_payload[\"filters\"]\n normalized_extra_form_data = normalized_filter_payload[\"extra_form_data\"]\n\n datasource_payload: Dict[str, Any] = {\n \"id\": dataset_id,\n \"type\": \"table\",\n }\n datasource = dataset_record.get(\"datasource\")\n if isinstance(datasource, dict):\n datasource_id = datasource.get(\"id\")\n datasource_type = datasource.get(\"type\")\n if datasource_id is not None:\n datasource_payload[\"id\"] = datasource_id\n if datasource_type:\n datasource_payload[\"type\"] = datasource_type\n\n serialized_dataset_template_params = dataset_record.get(\"template_params\")\n if (\n isinstance(serialized_dataset_template_params, str)\n and serialized_dataset_template_params.strip()\n ):\n try:\n parsed_dataset_template_params = json.loads(\n serialized_dataset_template_params\n )\n if isinstance(parsed_dataset_template_params, dict):\n for key, value in parsed_dataset_template_params.items():\n normalized_template_params.setdefault(str(key), value)\n except json.JSONDecodeError:\n app_logger.explore(\n \"Dataset template_params could not be parsed while building preview query context\",\n extra={\"dataset_id\": dataset_id},\n )\n\n extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data)\n if normalized_filters:\n extra_form_data[\"filters\"] = deepcopy(normalized_filters)\n\n query_object: Dict[str, Any] = {\n \"filters\": normalized_filters,\n \"extras\": {\"where\": \"\"},\n \"columns\": [],\n \"metrics\": [\"count\"],\n \"orderby\": [],\n \"annotation_layers\": [],\n \"row_limit\": 1000,\n \"series_limit\": 0,\n \"url_params\": normalized_template_params,\n \"applied_time_extras\": {},\n \"result_type\": \"query\",\n }\n\n schema = dataset_record.get(\"schema\")\n if schema:\n query_object[\"schema\"] = schema\n\n time_range = extra_form_data.get(\"time_range\") or dataset_record.get(\n \"default_time_range\"\n )\n if time_range:\n query_object[\"time_range\"] = time_range\n extra_form_data[\"time_range\"] = time_range\n\n result_format = dataset_record.get(\"result_format\") or \"json\"\n result_type = \"query\"\n\n form_data: Dict[str, Any] = {\n \"datasource\": f\"{datasource_payload['id']}__{datasource_payload['type']}\",\n \"datasource_id\": datasource_payload[\"id\"],\n \"datasource_type\": datasource_payload[\"type\"],\n \"viz_type\": \"table\",\n \"slice_id\": None,\n \"query_mode\": \"raw\",\n \"url_params\": normalized_template_params,\n \"extra_filters\": deepcopy(normalized_filters),\n \"adhoc_filters\": [],\n }\n if extra_form_data:\n form_data[\"extra_form_data\"] = extra_form_data\n\n payload = {\n \"datasource\": datasource_payload,\n \"queries\": [query_object],\n \"form_data\": form_data,\n \"result_format\": result_format,\n \"result_type\": result_type,\n \"force\": True,\n }\n app_logger.reflect(\n \"Built Superset dataset preview query context\",\n extra={\n \"dataset_id\": dataset_id,\n \"datasource\": datasource_payload,\n \"normalized_effective_filters\": normalized_filters,\n \"normalized_filter_diagnostics\": normalized_filter_payload[\n \"diagnostics\"\n ],\n \"result_type\": result_type,\n \"result_format\": result_format,\n },\n )\n return payload\n\n # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n\n# [/DEF:SupersetDatasetsPreviewMixin:Class]\n# [/DEF:SupersetDatasetsPreviewMixin:Module]\n" + "body": "# [DEF:SupersetDatasetsPreviewMixin:Module]\n# @COMPLEXITY: 4\n# @LAYER: Infra\n# @SEMANTICS: superset, datasets, preview, sql, compilation, filters\n# @PURPOSE: Dataset preview compilation mixin for SupersetClient — build query context, compile SQL.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDatasetsMixin]\n\nimport json\nfrom copy import deepcopy\nfrom typing import Any, Dict, List, Optional\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\nfrom ..utils.network import SupersetAPIError\n\nlog = MarkerLogger(\"SupersetDatasetsPreview\")\n\n# [DEF:SupersetDatasetsPreviewMixin:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Mixin providing dataset preview compilation and query context building.\nclass SupersetDatasetsPreviewMixin:\n # [DEF:SupersetClientCompileDatasetPreview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.\n def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:\n with belief_scope('SupersetClientCompileDatasetPreview'):\n log.reason('Compiling dataset preview SQL', payload={\"dataset_id\": dataset_id})\n dataset_response = self.get_dataset(dataset_id)\n dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {}\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str)\n request_payload = json.dumps(query_context)\n strategy_attempts: List[Dict[str, Any]] = []\n strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}]\n for candidate in strategy_candidates:\n endpoint_kind = candidate['endpoint_kind']\n endpoint_path = candidate['endpoint']\n request_transport = candidate['request_transport']\n request_params = deepcopy(candidate.get('params') or {})\n request_body = candidate.get('data')\n request_headers = deepcopy(candidate.get('headers') or {})\n request_param_keys = sorted(request_params.keys())\n request_payload_keys: List[str] = []\n if isinstance(request_body, str):\n try:\n decoded_request_body = json.loads(request_body)\n if isinstance(decoded_request_body, dict):\n request_payload_keys = sorted(decoded_request_body.keys())\n except json.JSONDecodeError:\n request_payload_keys = []\n elif isinstance(request_body, dict):\n request_payload_keys = sorted(request_body.keys())\n strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys}\n log.reason('Attempting dataset preview compilation strategy', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})\n try:\n response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None)\n normalized = self._extract_compiled_sql_from_preview_response(response)\n normalized['query_context'] = query_context\n normalized['legacy_form_data'] = legacy_form_data\n normalized['endpoint'] = endpoint_path\n normalized['endpoint_kind'] = endpoint_kind\n normalized['dataset_id'] = dataset_id\n normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}]\n log.reflect('Dataset preview compilation returned normalized SQL payload', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})\n return normalized\n except Exception as exc:\n failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)}\n strategy_attempts.append(failure_diagnostics)\n log.explore('Dataset preview compilation strategy failed', payload={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}, error=str(exc))\n raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})')\n # [/DEF:SupersetClientCompileDatasetPreview:Function]\n\n # [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.\n def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'):\n log.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters)\n query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {})\n legacy_form_data = deepcopy(query_context.get('form_data', {}))\n legacy_form_data.pop('datasource', None)\n legacy_form_data['metrics'] = deepcopy(query_object.get('metrics', ['count']))\n legacy_form_data['columns'] = deepcopy(query_object.get('columns', []))\n legacy_form_data['orderby'] = deepcopy(query_object.get('orderby', []))\n legacy_form_data['annotation_layers'] = deepcopy(query_object.get('annotation_layers', []))\n legacy_form_data['row_limit'] = query_object.get('row_limit', 1000)\n legacy_form_data['series_limit'] = query_object.get('series_limit', 0)\n legacy_form_data['url_params'] = deepcopy(query_object.get('url_params', template_params))\n legacy_form_data['applied_time_extras'] = deepcopy(query_object.get('applied_time_extras', {}))\n legacy_form_data['result_format'] = query_context.get('result_format', 'json')\n legacy_form_data['result_type'] = query_context.get('result_type', 'query')\n legacy_form_data['force'] = bool(query_context.get('force', True))\n extras = query_object.get('extras')\n if isinstance(extras, dict):\n legacy_form_data['extras'] = deepcopy(extras)\n time_range = query_object.get('time_range')\n if time_range:\n legacy_form_data['time_range'] = time_range\n log.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', payload={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})\n log.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n return legacy_form_data\n # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n\n # [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.\n def build_dataset_preview_query_context(\n self,\n dataset_id: int,\n dataset_record: Dict[str, Any],\n template_params: Dict[str, Any],\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetClientBuildDatasetPreviewQueryContext\"):\n log.reason(\n \"Building Superset dataset preview query context\",\n payload={\"dataset_id\": dataset_id, \"filter_count\": len(effective_filters or [])},\n )\n normalized_template_params = deepcopy(template_params or {})\n normalized_filter_payload = (\n self._normalize_effective_filters_for_query_context(\n effective_filters or []\n )\n )\n normalized_filters = normalized_filter_payload[\"filters\"]\n normalized_extra_form_data = normalized_filter_payload[\"extra_form_data\"]\n\n datasource_payload: Dict[str, Any] = {\n \"id\": dataset_id,\n \"type\": \"table\",\n }\n datasource = dataset_record.get(\"datasource\")\n if isinstance(datasource, dict):\n datasource_id = datasource.get(\"id\")\n datasource_type = datasource.get(\"type\")\n if datasource_id is not None:\n datasource_payload[\"id\"] = datasource_id\n if datasource_type:\n datasource_payload[\"type\"] = datasource_type\n\n serialized_dataset_template_params = dataset_record.get(\"template_params\")\n if (\n isinstance(serialized_dataset_template_params, str)\n and serialized_dataset_template_params.strip()\n ):\n try:\n parsed_dataset_template_params = json.loads(\n serialized_dataset_template_params\n )\n if isinstance(parsed_dataset_template_params, dict):\n for key, value in parsed_dataset_template_params.items():\n normalized_template_params.setdefault(str(key), value)\n except json.JSONDecodeError:\n log.explore(\"Dataset template_params could not be parsed while building preview query context\", error=\"Failed to parse template_params JSON\", payload={\"dataset_id\": dataset_id})\n\n extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data)\n if normalized_filters:\n extra_form_data[\"filters\"] = deepcopy(normalized_filters)\n\n query_object: Dict[str, Any] = {\n \"filters\": normalized_filters,\n \"extras\": {\"where\": \"\"},\n \"columns\": [],\n \"metrics\": [\"count\"],\n \"orderby\": [],\n \"annotation_layers\": [],\n \"row_limit\": 1000,\n \"series_limit\": 0,\n \"url_params\": normalized_template_params,\n \"applied_time_extras\": {},\n \"result_type\": \"query\",\n }\n\n schema = dataset_record.get(\"schema\")\n if schema:\n query_object[\"schema\"] = schema\n\n time_range = extra_form_data.get(\"time_range\") or dataset_record.get(\n \"default_time_range\"\n )\n if time_range:\n query_object[\"time_range\"] = time_range\n extra_form_data[\"time_range\"] = time_range\n\n result_format = dataset_record.get(\"result_format\") or \"json\"\n result_type = \"query\"\n\n form_data: Dict[str, Any] = {\n \"datasource\": f\"{datasource_payload['id']}__{datasource_payload['type']}\",\n \"datasource_id\": datasource_payload[\"id\"],\n \"datasource_type\": datasource_payload[\"type\"],\n \"viz_type\": \"table\",\n \"slice_id\": None,\n \"query_mode\": \"raw\",\n \"url_params\": normalized_template_params,\n \"extra_filters\": deepcopy(normalized_filters),\n \"adhoc_filters\": [],\n }\n if extra_form_data:\n form_data[\"extra_form_data\"] = extra_form_data\n\n payload = {\n \"datasource\": datasource_payload,\n \"queries\": [query_object],\n \"form_data\": form_data,\n \"result_format\": result_format,\n \"result_type\": result_type,\n \"force\": True,\n }\n log.reflect(\n \"Built Superset dataset preview query context\",\n payload={\n \"dataset_id\": dataset_id,\n \"datasource\": datasource_payload,\n \"normalized_effective_filters\": normalized_filters,\n \"normalized_filter_diagnostics\": normalized_filter_payload[\n \"diagnostics\"\n ],\n \"result_type\": result_type,\n \"result_format\": result_format,\n },\n )\n return payload\n\n # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n\n# [/DEF:SupersetDatasetsPreviewMixin:Class]\n# [/DEF:SupersetDatasetsPreviewMixin:Module]\n" }, { "contract_id": "SupersetClientCompileDatasetPreview", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets_preview.py", - "start_line": 19, - "end_line": 70, + "start_line": 23, + "end_line": 73, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -40447,14 +41203,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientCompileDatasetPreview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.\n def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:\n with belief_scope('SupersetClientCompileDatasetPreview'):\n app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview')\n dataset_response = self.get_dataset(dataset_id)\n dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {}\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str)\n request_payload = json.dumps(query_context)\n strategy_attempts: List[Dict[str, Any]] = []\n strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}]\n for candidate in strategy_candidates:\n endpoint_kind = candidate['endpoint_kind']\n endpoint_path = candidate['endpoint']\n request_transport = candidate['request_transport']\n request_params = deepcopy(candidate.get('params') or {})\n request_body = candidate.get('data')\n request_headers = deepcopy(candidate.get('headers') or {})\n request_param_keys = sorted(request_params.keys())\n request_payload_keys: List[str] = []\n if isinstance(request_body, str):\n try:\n decoded_request_body = json.loads(request_body)\n if isinstance(decoded_request_body, dict):\n request_payload_keys = sorted(decoded_request_body.keys())\n except json.JSONDecodeError:\n request_payload_keys = []\n elif isinstance(request_body, dict):\n request_payload_keys = sorted(request_body.keys())\n strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys}\n app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})\n try:\n response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None)\n normalized = self._extract_compiled_sql_from_preview_response(response)\n normalized['query_context'] = query_context\n normalized['legacy_form_data'] = legacy_form_data\n normalized['endpoint'] = endpoint_path\n normalized['endpoint_kind'] = endpoint_kind\n normalized['dataset_id'] = dataset_id\n normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}]\n app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})\n app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview')\n return normalized\n except Exception as exc:\n failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)}\n strategy_attempts.append(failure_diagnostics)\n app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body})\n raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})')\n # [/DEF:SupersetClientCompileDatasetPreview:Function]\n" + "body": " # [DEF:SupersetClientCompileDatasetPreview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.\n def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:\n with belief_scope('SupersetClientCompileDatasetPreview'):\n log.reason('Compiling dataset preview SQL', payload={\"dataset_id\": dataset_id})\n dataset_response = self.get_dataset(dataset_id)\n dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {}\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])\n legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str)\n request_payload = json.dumps(query_context)\n strategy_attempts: List[Dict[str, Any]] = []\n strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}]\n for candidate in strategy_candidates:\n endpoint_kind = candidate['endpoint_kind']\n endpoint_path = candidate['endpoint']\n request_transport = candidate['request_transport']\n request_params = deepcopy(candidate.get('params') or {})\n request_body = candidate.get('data')\n request_headers = deepcopy(candidate.get('headers') or {})\n request_param_keys = sorted(request_params.keys())\n request_payload_keys: List[str] = []\n if isinstance(request_body, str):\n try:\n decoded_request_body = json.loads(request_body)\n if isinstance(decoded_request_body, dict):\n request_payload_keys = sorted(decoded_request_body.keys())\n except json.JSONDecodeError:\n request_payload_keys = []\n elif isinstance(request_body, dict):\n request_payload_keys = sorted(request_body.keys())\n strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys}\n log.reason('Attempting dataset preview compilation strategy', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})\n try:\n response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None)\n normalized = self._extract_compiled_sql_from_preview_response(response)\n normalized['query_context'] = query_context\n normalized['legacy_form_data'] = legacy_form_data\n normalized['endpoint'] = endpoint_path\n normalized['endpoint_kind'] = endpoint_kind\n normalized['dataset_id'] = dataset_id\n normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}]\n log.reflect('Dataset preview compilation returned normalized SQL payload', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})\n return normalized\n except Exception as exc:\n failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)}\n strategy_attempts.append(failure_diagnostics)\n log.explore('Dataset preview compilation strategy failed', payload={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}, error=str(exc))\n raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})')\n # [/DEF:SupersetClientCompileDatasetPreview:Function]\n" }, { "contract_id": "SupersetClientBuildDatasetPreviewLegacyFormData", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets_preview.py", - "start_line": 72, - "end_line": 102, + "start_line": 75, + "end_line": 105, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -40501,13 +41257,13 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.\n def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'):\n app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters)\n query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {})\n legacy_form_data = deepcopy(query_context.get('form_data', {}))\n legacy_form_data.pop('datasource', None)\n legacy_form_data['metrics'] = deepcopy(query_object.get('metrics', ['count']))\n legacy_form_data['columns'] = deepcopy(query_object.get('columns', []))\n legacy_form_data['orderby'] = deepcopy(query_object.get('orderby', []))\n legacy_form_data['annotation_layers'] = deepcopy(query_object.get('annotation_layers', []))\n legacy_form_data['row_limit'] = query_object.get('row_limit', 1000)\n legacy_form_data['series_limit'] = query_object.get('series_limit', 0)\n legacy_form_data['url_params'] = deepcopy(query_object.get('url_params', template_params))\n legacy_form_data['applied_time_extras'] = deepcopy(query_object.get('applied_time_extras', {}))\n legacy_form_data['result_format'] = query_context.get('result_format', 'json')\n legacy_form_data['result_type'] = query_context.get('result_type', 'query')\n legacy_form_data['force'] = bool(query_context.get('force', True))\n extras = query_object.get('extras')\n if isinstance(extras, dict):\n legacy_form_data['extras'] = deepcopy(extras)\n time_range = query_object.get('time_range')\n if time_range:\n legacy_form_data['time_range'] = time_range\n app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})\n app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n return legacy_form_data\n # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n" + "body": " # [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.\n def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]:\n with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'):\n log.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters)\n query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {})\n legacy_form_data = deepcopy(query_context.get('form_data', {}))\n legacy_form_data.pop('datasource', None)\n legacy_form_data['metrics'] = deepcopy(query_object.get('metrics', ['count']))\n legacy_form_data['columns'] = deepcopy(query_object.get('columns', []))\n legacy_form_data['orderby'] = deepcopy(query_object.get('orderby', []))\n legacy_form_data['annotation_layers'] = deepcopy(query_object.get('annotation_layers', []))\n legacy_form_data['row_limit'] = query_object.get('row_limit', 1000)\n legacy_form_data['series_limit'] = query_object.get('series_limit', 0)\n legacy_form_data['url_params'] = deepcopy(query_object.get('url_params', template_params))\n legacy_form_data['applied_time_extras'] = deepcopy(query_object.get('applied_time_extras', {}))\n legacy_form_data['result_format'] = query_context.get('result_format', 'json')\n legacy_form_data['result_type'] = query_context.get('result_type', 'query')\n legacy_form_data['force'] = bool(query_context.get('force', True))\n extras = query_object.get('extras')\n if isinstance(extras, dict):\n legacy_form_data['extras'] = deepcopy(extras)\n time_range = query_object.get('time_range')\n if time_range:\n legacy_form_data['time_range'] = time_range\n log.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', payload={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})\n log.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')\n return legacy_form_data\n # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]\n" }, { "contract_id": "SupersetClientBuildDatasetPreviewQueryContext", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets_preview.py", - "start_line": 104, + "start_line": 107, "end_line": 228, "tier": "TIER_2", "complexity": 4, @@ -40555,14 +41311,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.\n def build_dataset_preview_query_context(\n self,\n dataset_id: int,\n dataset_record: Dict[str, Any],\n template_params: Dict[str, Any],\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetClientBuildDatasetPreviewQueryContext\"):\n app_logger.reason(\n \"Building Superset dataset preview query context\",\n extra={\"dataset_id\": dataset_id, \"filter_count\": len(effective_filters or [])},\n )\n normalized_template_params = deepcopy(template_params or {})\n normalized_filter_payload = (\n self._normalize_effective_filters_for_query_context(\n effective_filters or []\n )\n )\n normalized_filters = normalized_filter_payload[\"filters\"]\n normalized_extra_form_data = normalized_filter_payload[\"extra_form_data\"]\n\n datasource_payload: Dict[str, Any] = {\n \"id\": dataset_id,\n \"type\": \"table\",\n }\n datasource = dataset_record.get(\"datasource\")\n if isinstance(datasource, dict):\n datasource_id = datasource.get(\"id\")\n datasource_type = datasource.get(\"type\")\n if datasource_id is not None:\n datasource_payload[\"id\"] = datasource_id\n if datasource_type:\n datasource_payload[\"type\"] = datasource_type\n\n serialized_dataset_template_params = dataset_record.get(\"template_params\")\n if (\n isinstance(serialized_dataset_template_params, str)\n and serialized_dataset_template_params.strip()\n ):\n try:\n parsed_dataset_template_params = json.loads(\n serialized_dataset_template_params\n )\n if isinstance(parsed_dataset_template_params, dict):\n for key, value in parsed_dataset_template_params.items():\n normalized_template_params.setdefault(str(key), value)\n except json.JSONDecodeError:\n app_logger.explore(\n \"Dataset template_params could not be parsed while building preview query context\",\n extra={\"dataset_id\": dataset_id},\n )\n\n extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data)\n if normalized_filters:\n extra_form_data[\"filters\"] = deepcopy(normalized_filters)\n\n query_object: Dict[str, Any] = {\n \"filters\": normalized_filters,\n \"extras\": {\"where\": \"\"},\n \"columns\": [],\n \"metrics\": [\"count\"],\n \"orderby\": [],\n \"annotation_layers\": [],\n \"row_limit\": 1000,\n \"series_limit\": 0,\n \"url_params\": normalized_template_params,\n \"applied_time_extras\": {},\n \"result_type\": \"query\",\n }\n\n schema = dataset_record.get(\"schema\")\n if schema:\n query_object[\"schema\"] = schema\n\n time_range = extra_form_data.get(\"time_range\") or dataset_record.get(\n \"default_time_range\"\n )\n if time_range:\n query_object[\"time_range\"] = time_range\n extra_form_data[\"time_range\"] = time_range\n\n result_format = dataset_record.get(\"result_format\") or \"json\"\n result_type = \"query\"\n\n form_data: Dict[str, Any] = {\n \"datasource\": f\"{datasource_payload['id']}__{datasource_payload['type']}\",\n \"datasource_id\": datasource_payload[\"id\"],\n \"datasource_type\": datasource_payload[\"type\"],\n \"viz_type\": \"table\",\n \"slice_id\": None,\n \"query_mode\": \"raw\",\n \"url_params\": normalized_template_params,\n \"extra_filters\": deepcopy(normalized_filters),\n \"adhoc_filters\": [],\n }\n if extra_form_data:\n form_data[\"extra_form_data\"] = extra_form_data\n\n payload = {\n \"datasource\": datasource_payload,\n \"queries\": [query_object],\n \"form_data\": form_data,\n \"result_format\": result_format,\n \"result_type\": result_type,\n \"force\": True,\n }\n app_logger.reflect(\n \"Built Superset dataset preview query context\",\n extra={\n \"dataset_id\": dataset_id,\n \"datasource\": datasource_payload,\n \"normalized_effective_filters\": normalized_filters,\n \"normalized_filter_diagnostics\": normalized_filter_payload[\n \"diagnostics\"\n ],\n \"result_type\": result_type,\n \"result_format\": result_format,\n },\n )\n return payload\n\n # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n" + "body": " # [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.\n def build_dataset_preview_query_context(\n self,\n dataset_id: int,\n dataset_record: Dict[str, Any],\n template_params: Dict[str, Any],\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetClientBuildDatasetPreviewQueryContext\"):\n log.reason(\n \"Building Superset dataset preview query context\",\n payload={\"dataset_id\": dataset_id, \"filter_count\": len(effective_filters or [])},\n )\n normalized_template_params = deepcopy(template_params or {})\n normalized_filter_payload = (\n self._normalize_effective_filters_for_query_context(\n effective_filters or []\n )\n )\n normalized_filters = normalized_filter_payload[\"filters\"]\n normalized_extra_form_data = normalized_filter_payload[\"extra_form_data\"]\n\n datasource_payload: Dict[str, Any] = {\n \"id\": dataset_id,\n \"type\": \"table\",\n }\n datasource = dataset_record.get(\"datasource\")\n if isinstance(datasource, dict):\n datasource_id = datasource.get(\"id\")\n datasource_type = datasource.get(\"type\")\n if datasource_id is not None:\n datasource_payload[\"id\"] = datasource_id\n if datasource_type:\n datasource_payload[\"type\"] = datasource_type\n\n serialized_dataset_template_params = dataset_record.get(\"template_params\")\n if (\n isinstance(serialized_dataset_template_params, str)\n and serialized_dataset_template_params.strip()\n ):\n try:\n parsed_dataset_template_params = json.loads(\n serialized_dataset_template_params\n )\n if isinstance(parsed_dataset_template_params, dict):\n for key, value in parsed_dataset_template_params.items():\n normalized_template_params.setdefault(str(key), value)\n except json.JSONDecodeError:\n log.explore(\"Dataset template_params could not be parsed while building preview query context\", error=\"Failed to parse template_params JSON\", payload={\"dataset_id\": dataset_id})\n\n extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data)\n if normalized_filters:\n extra_form_data[\"filters\"] = deepcopy(normalized_filters)\n\n query_object: Dict[str, Any] = {\n \"filters\": normalized_filters,\n \"extras\": {\"where\": \"\"},\n \"columns\": [],\n \"metrics\": [\"count\"],\n \"orderby\": [],\n \"annotation_layers\": [],\n \"row_limit\": 1000,\n \"series_limit\": 0,\n \"url_params\": normalized_template_params,\n \"applied_time_extras\": {},\n \"result_type\": \"query\",\n }\n\n schema = dataset_record.get(\"schema\")\n if schema:\n query_object[\"schema\"] = schema\n\n time_range = extra_form_data.get(\"time_range\") or dataset_record.get(\n \"default_time_range\"\n )\n if time_range:\n query_object[\"time_range\"] = time_range\n extra_form_data[\"time_range\"] = time_range\n\n result_format = dataset_record.get(\"result_format\") or \"json\"\n result_type = \"query\"\n\n form_data: Dict[str, Any] = {\n \"datasource\": f\"{datasource_payload['id']}__{datasource_payload['type']}\",\n \"datasource_id\": datasource_payload[\"id\"],\n \"datasource_type\": datasource_payload[\"type\"],\n \"viz_type\": \"table\",\n \"slice_id\": None,\n \"query_mode\": \"raw\",\n \"url_params\": normalized_template_params,\n \"extra_filters\": deepcopy(normalized_filters),\n \"adhoc_filters\": [],\n }\n if extra_form_data:\n form_data[\"extra_form_data\"] = extra_form_data\n\n payload = {\n \"datasource\": datasource_payload,\n \"queries\": [query_object],\n \"form_data\": form_data,\n \"result_format\": result_format,\n \"result_type\": result_type,\n \"force\": True,\n }\n log.reflect(\n \"Built Superset dataset preview query context\",\n payload={\n \"dataset_id\": dataset_id,\n \"datasource\": datasource_payload,\n \"normalized_effective_filters\": normalized_filters,\n \"normalized_filter_diagnostics\": normalized_filter_payload[\n \"diagnostics\"\n ],\n \"result_type\": result_type,\n \"result_format\": result_format,\n },\n )\n return payload\n\n # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]\n" }, { "contract_id": "SupersetDatasetsPreviewFiltersMixin", "contract_type": "Module", "file_path": "backend/src/core/superset_client/_datasets_preview_filters.py", "start_line": 1, - "end_line": 201, + "end_line": 203, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40595,14 +41351,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:SupersetDatasetsPreviewFiltersMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, datasets, preview, filters, normalization, sql, extraction\n# @PURPOSE: Filter normalization and SQL extraction helpers for dataset preview compilation.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin]\n\nfrom copy import deepcopy\nfrom typing import Any, Dict, List, Optional, Tuple\n\nfrom ..logger import logger as app_logger, belief_scope\nfrom ..utils.network import SupersetAPIError\n\n\n# [DEF:SupersetDatasetsPreviewFiltersMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin]\nclass SupersetDatasetsPreviewFiltersMixin:\n # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Convert execution mappings into Superset chart-data filter objects.\n def _normalize_effective_filters_for_query_context(\n self,\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\n \"SupersetClient._normalize_effective_filters_for_query_context\"\n ):\n normalized_filters: List[Dict[str, Any]] = []\n merged_extra_form_data: Dict[str, Any] = {}\n diagnostics: List[Dict[str, Any]] = []\n\n for item in effective_filters:\n if not isinstance(item, dict):\n continue\n\n display_name = str(\n item.get(\"display_name\")\n or item.get(\"filter_name\")\n or item.get(\"variable_name\")\n or \"unresolved_filter\"\n ).strip()\n value = item.get(\"effective_value\")\n normalized_payload = item.get(\"normalized_filter_payload\")\n preserved_clauses: List[Dict[str, Any]] = []\n preserved_extra_form_data: Dict[str, Any] = {}\n used_preserved_clauses = False\n\n if isinstance(normalized_payload, dict):\n raw_clauses = normalized_payload.get(\"filter_clauses\")\n if isinstance(raw_clauses, list):\n preserved_clauses = [\n deepcopy(clause)\n for clause in raw_clauses\n if isinstance(clause, dict)\n ]\n raw_extra_form_data = normalized_payload.get(\"extra_form_data\")\n if isinstance(raw_extra_form_data, dict):\n preserved_extra_form_data = deepcopy(raw_extra_form_data)\n\n if isinstance(preserved_extra_form_data, dict):\n for key, extra_value in preserved_extra_form_data.items():\n if key == \"filters\":\n continue\n merged_extra_form_data[key] = deepcopy(extra_value)\n\n outgoing_clauses: List[Dict[str, Any]] = []\n if preserved_clauses:\n for clause in preserved_clauses:\n clause_copy = deepcopy(clause)\n if \"val\" not in clause_copy and value is not None:\n clause_copy[\"val\"] = deepcopy(value)\n outgoing_clauses.append(clause_copy)\n used_preserved_clauses = True\n elif preserved_extra_form_data:\n outgoing_clauses = []\n else:\n column = str(\n item.get(\"variable_name\") or item.get(\"filter_name\") or \"\"\n ).strip()\n if column and value is not None:\n operator = \"IN\" if isinstance(value, list) else \"==\"\n outgoing_clauses.append(\n {\"col\": column, \"op\": operator, \"val\": value}\n )\n\n normalized_filters.extend(outgoing_clauses)\n diagnostics.append(\n {\n \"filter_name\": display_name,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else None\n ),\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": deepcopy(outgoing_clauses),\n }\n )\n app_logger.reason(\n \"Normalized effective preview filter for Superset query context\",\n extra={\n \"filter_name\": display_name,\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": outgoing_clauses,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else \"heuristic_reconstruction\"\n ),\n },\n )\n\n return {\n \"filters\": normalized_filters,\n \"extra_form_data\": merged_extra_form_data,\n \"diagnostics\": diagnostics,\n }\n\n # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n\n # [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses.\n def _extract_compiled_sql_from_preview_response(\n self, response: Any\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetClient._extract_compiled_sql_from_preview_response\"):\n if not isinstance(response, dict):\n raise SupersetAPIError(\n \"Superset preview response was not a JSON object\"\n )\n\n response_diagnostics: List[Dict[str, Any]] = []\n result_payload = response.get(\"result\")\n if isinstance(result_payload, list):\n for index, item in enumerate(result_payload):\n if not isinstance(item, dict):\n continue\n compiled_sql = str(\n item.get(\"query\")\n or item.get(\"sql\")\n or item.get(\"compiled_sql\")\n or \"\"\n ).strip()\n response_diagnostics.append(\n {\n \"index\": index,\n \"status\": item.get(\"status\"),\n \"applied_filters\": item.get(\"applied_filters\"),\n \"rejected_filters\": item.get(\"rejected_filters\"),\n \"has_query\": bool(compiled_sql),\n \"source\": \"result_list\",\n }\n )\n if compiled_sql:\n return {\n \"compiled_sql\": compiled_sql,\n \"raw_response\": response,\n \"response_diagnostics\": response_diagnostics,\n }\n\n top_level_candidates: List[Tuple[str, Any]] = [\n (\"query\", response.get(\"query\")),\n (\"sql\", response.get(\"sql\")),\n (\"compiled_sql\", response.get(\"compiled_sql\")),\n ]\n if isinstance(result_payload, dict):\n top_level_candidates.extend(\n [\n (\"result.query\", result_payload.get(\"query\")),\n (\"result.sql\", result_payload.get(\"sql\")),\n (\"result.compiled_sql\", result_payload.get(\"compiled_sql\")),\n ]\n )\n\n for source, candidate in top_level_candidates:\n compiled_sql = str(candidate or \"\").strip()\n response_diagnostics.append(\n {\"source\": source, \"has_query\": bool(compiled_sql)}\n )\n if compiled_sql:\n return {\n \"compiled_sql\": compiled_sql,\n \"raw_response\": response,\n \"response_diagnostics\": response_diagnostics,\n }\n\n raise SupersetAPIError(\n \"Superset preview response did not expose compiled SQL \"\n f\"(diagnostics={response_diagnostics!r})\"\n )\n\n # [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]\n\n\n# [/DEF:SupersetDatasetsPreviewFiltersMixin:Class]\n# [/DEF:SupersetDatasetsPreviewFiltersMixin:Module]\n" + "body": "# [DEF:SupersetDatasetsPreviewFiltersMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: superset, datasets, preview, filters, normalization, sql, extraction\n# @PURPOSE: Filter normalization and SQL extraction helpers for dataset preview compilation.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin]\n\nfrom copy import deepcopy\nfrom typing import Any, Dict, List, Optional, Tuple\n\nfrom ..cot_logger import MarkerLogger\nfrom ..logger import belief_scope\nfrom ..utils.network import SupersetAPIError\n\nlog = MarkerLogger(\"SupersetDatasetsPreviewFilters\")\n\n# [DEF:SupersetDatasetsPreviewFiltersMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations.\n# @RELATION: DEPENDS_ON -> [SupersetClientBase]\n# @RELATION: DEPENDS_ON -> [SupersetDatasetsPreviewMixin]\nclass SupersetDatasetsPreviewFiltersMixin:\n # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Convert execution mappings into Superset chart-data filter objects.\n def _normalize_effective_filters_for_query_context(\n self,\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\n \"SupersetClient._normalize_effective_filters_for_query_context\"\n ):\n normalized_filters: List[Dict[str, Any]] = []\n merged_extra_form_data: Dict[str, Any] = {}\n diagnostics: List[Dict[str, Any]] = []\n\n for item in effective_filters:\n if not isinstance(item, dict):\n continue\n\n display_name = str(\n item.get(\"display_name\")\n or item.get(\"filter_name\")\n or item.get(\"variable_name\")\n or \"unresolved_filter\"\n ).strip()\n value = item.get(\"effective_value\")\n normalized_payload = item.get(\"normalized_filter_payload\")\n preserved_clauses: List[Dict[str, Any]] = []\n preserved_extra_form_data: Dict[str, Any] = {}\n used_preserved_clauses = False\n\n if isinstance(normalized_payload, dict):\n raw_clauses = normalized_payload.get(\"filter_clauses\")\n if isinstance(raw_clauses, list):\n preserved_clauses = [\n deepcopy(clause)\n for clause in raw_clauses\n if isinstance(clause, dict)\n ]\n raw_extra_form_data = normalized_payload.get(\"extra_form_data\")\n if isinstance(raw_extra_form_data, dict):\n preserved_extra_form_data = deepcopy(raw_extra_form_data)\n\n if isinstance(preserved_extra_form_data, dict):\n for key, extra_value in preserved_extra_form_data.items():\n if key == \"filters\":\n continue\n merged_extra_form_data[key] = deepcopy(extra_value)\n\n outgoing_clauses: List[Dict[str, Any]] = []\n if preserved_clauses:\n for clause in preserved_clauses:\n clause_copy = deepcopy(clause)\n if \"val\" not in clause_copy and value is not None:\n clause_copy[\"val\"] = deepcopy(value)\n outgoing_clauses.append(clause_copy)\n used_preserved_clauses = True\n elif preserved_extra_form_data:\n outgoing_clauses = []\n else:\n column = str(\n item.get(\"variable_name\") or item.get(\"filter_name\") or \"\"\n ).strip()\n if column and value is not None:\n operator = \"IN\" if isinstance(value, list) else \"==\"\n outgoing_clauses.append(\n {\"col\": column, \"op\": operator, \"val\": value}\n )\n\n normalized_filters.extend(outgoing_clauses)\n diagnostics.append(\n {\n \"filter_name\": display_name,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else None\n ),\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": deepcopy(outgoing_clauses),\n }\n )\n log.reason(\n \"Normalized effective preview filter for Superset query context\",\n payload={\n \"filter_name\": display_name,\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": outgoing_clauses,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else \"heuristic_reconstruction\"\n ),\n },\n )\n\n return {\n \"filters\": normalized_filters,\n \"extra_form_data\": merged_extra_form_data,\n \"diagnostics\": diagnostics,\n }\n\n # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n\n # [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses.\n def _extract_compiled_sql_from_preview_response(\n self, response: Any\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetClient._extract_compiled_sql_from_preview_response\"):\n if not isinstance(response, dict):\n raise SupersetAPIError(\n \"Superset preview response was not a JSON object\"\n )\n\n response_diagnostics: List[Dict[str, Any]] = []\n result_payload = response.get(\"result\")\n if isinstance(result_payload, list):\n for index, item in enumerate(result_payload):\n if not isinstance(item, dict):\n continue\n compiled_sql = str(\n item.get(\"query\")\n or item.get(\"sql\")\n or item.get(\"compiled_sql\")\n or \"\"\n ).strip()\n response_diagnostics.append(\n {\n \"index\": index,\n \"status\": item.get(\"status\"),\n \"applied_filters\": item.get(\"applied_filters\"),\n \"rejected_filters\": item.get(\"rejected_filters\"),\n \"has_query\": bool(compiled_sql),\n \"source\": \"result_list\",\n }\n )\n if compiled_sql:\n return {\n \"compiled_sql\": compiled_sql,\n \"raw_response\": response,\n \"response_diagnostics\": response_diagnostics,\n }\n\n top_level_candidates: List[Tuple[str, Any]] = [\n (\"query\", response.get(\"query\")),\n (\"sql\", response.get(\"sql\")),\n (\"compiled_sql\", response.get(\"compiled_sql\")),\n ]\n if isinstance(result_payload, dict):\n top_level_candidates.extend(\n [\n (\"result.query\", result_payload.get(\"query\")),\n (\"result.sql\", result_payload.get(\"sql\")),\n (\"result.compiled_sql\", result_payload.get(\"compiled_sql\")),\n ]\n )\n\n for source, candidate in top_level_candidates:\n compiled_sql = str(candidate or \"\").strip()\n response_diagnostics.append(\n {\"source\": source, \"has_query\": bool(compiled_sql)}\n )\n if compiled_sql:\n return {\n \"compiled_sql\": compiled_sql,\n \"raw_response\": response,\n \"response_diagnostics\": response_diagnostics,\n }\n\n raise SupersetAPIError(\n \"Superset preview response did not expose compiled SQL \"\n f\"(diagnostics={response_diagnostics!r})\"\n )\n\n # [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]\n\n\n# [/DEF:SupersetDatasetsPreviewFiltersMixin:Class]\n# [/DEF:SupersetDatasetsPreviewFiltersMixin:Module]\n" }, { "contract_id": "SupersetClientNormalizeEffectiveFiltersForQueryContext", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets_preview_filters.py", - "start_line": 22, - "end_line": 123, + "start_line": 24, + "end_line": 125, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -40622,14 +41378,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Convert execution mappings into Superset chart-data filter objects.\n def _normalize_effective_filters_for_query_context(\n self,\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\n \"SupersetClient._normalize_effective_filters_for_query_context\"\n ):\n normalized_filters: List[Dict[str, Any]] = []\n merged_extra_form_data: Dict[str, Any] = {}\n diagnostics: List[Dict[str, Any]] = []\n\n for item in effective_filters:\n if not isinstance(item, dict):\n continue\n\n display_name = str(\n item.get(\"display_name\")\n or item.get(\"filter_name\")\n or item.get(\"variable_name\")\n or \"unresolved_filter\"\n ).strip()\n value = item.get(\"effective_value\")\n normalized_payload = item.get(\"normalized_filter_payload\")\n preserved_clauses: List[Dict[str, Any]] = []\n preserved_extra_form_data: Dict[str, Any] = {}\n used_preserved_clauses = False\n\n if isinstance(normalized_payload, dict):\n raw_clauses = normalized_payload.get(\"filter_clauses\")\n if isinstance(raw_clauses, list):\n preserved_clauses = [\n deepcopy(clause)\n for clause in raw_clauses\n if isinstance(clause, dict)\n ]\n raw_extra_form_data = normalized_payload.get(\"extra_form_data\")\n if isinstance(raw_extra_form_data, dict):\n preserved_extra_form_data = deepcopy(raw_extra_form_data)\n\n if isinstance(preserved_extra_form_data, dict):\n for key, extra_value in preserved_extra_form_data.items():\n if key == \"filters\":\n continue\n merged_extra_form_data[key] = deepcopy(extra_value)\n\n outgoing_clauses: List[Dict[str, Any]] = []\n if preserved_clauses:\n for clause in preserved_clauses:\n clause_copy = deepcopy(clause)\n if \"val\" not in clause_copy and value is not None:\n clause_copy[\"val\"] = deepcopy(value)\n outgoing_clauses.append(clause_copy)\n used_preserved_clauses = True\n elif preserved_extra_form_data:\n outgoing_clauses = []\n else:\n column = str(\n item.get(\"variable_name\") or item.get(\"filter_name\") or \"\"\n ).strip()\n if column and value is not None:\n operator = \"IN\" if isinstance(value, list) else \"==\"\n outgoing_clauses.append(\n {\"col\": column, \"op\": operator, \"val\": value}\n )\n\n normalized_filters.extend(outgoing_clauses)\n diagnostics.append(\n {\n \"filter_name\": display_name,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else None\n ),\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": deepcopy(outgoing_clauses),\n }\n )\n app_logger.reason(\n \"Normalized effective preview filter for Superset query context\",\n extra={\n \"filter_name\": display_name,\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": outgoing_clauses,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else \"heuristic_reconstruction\"\n ),\n },\n )\n\n return {\n \"filters\": normalized_filters,\n \"extra_form_data\": merged_extra_form_data,\n \"diagnostics\": diagnostics,\n }\n\n # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n" + "body": " # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Convert execution mappings into Superset chart-data filter objects.\n def _normalize_effective_filters_for_query_context(\n self,\n effective_filters: List[Dict[str, Any]],\n ) -> Dict[str, Any]:\n with belief_scope(\n \"SupersetClient._normalize_effective_filters_for_query_context\"\n ):\n normalized_filters: List[Dict[str, Any]] = []\n merged_extra_form_data: Dict[str, Any] = {}\n diagnostics: List[Dict[str, Any]] = []\n\n for item in effective_filters:\n if not isinstance(item, dict):\n continue\n\n display_name = str(\n item.get(\"display_name\")\n or item.get(\"filter_name\")\n or item.get(\"variable_name\")\n or \"unresolved_filter\"\n ).strip()\n value = item.get(\"effective_value\")\n normalized_payload = item.get(\"normalized_filter_payload\")\n preserved_clauses: List[Dict[str, Any]] = []\n preserved_extra_form_data: Dict[str, Any] = {}\n used_preserved_clauses = False\n\n if isinstance(normalized_payload, dict):\n raw_clauses = normalized_payload.get(\"filter_clauses\")\n if isinstance(raw_clauses, list):\n preserved_clauses = [\n deepcopy(clause)\n for clause in raw_clauses\n if isinstance(clause, dict)\n ]\n raw_extra_form_data = normalized_payload.get(\"extra_form_data\")\n if isinstance(raw_extra_form_data, dict):\n preserved_extra_form_data = deepcopy(raw_extra_form_data)\n\n if isinstance(preserved_extra_form_data, dict):\n for key, extra_value in preserved_extra_form_data.items():\n if key == \"filters\":\n continue\n merged_extra_form_data[key] = deepcopy(extra_value)\n\n outgoing_clauses: List[Dict[str, Any]] = []\n if preserved_clauses:\n for clause in preserved_clauses:\n clause_copy = deepcopy(clause)\n if \"val\" not in clause_copy and value is not None:\n clause_copy[\"val\"] = deepcopy(value)\n outgoing_clauses.append(clause_copy)\n used_preserved_clauses = True\n elif preserved_extra_form_data:\n outgoing_clauses = []\n else:\n column = str(\n item.get(\"variable_name\") or item.get(\"filter_name\") or \"\"\n ).strip()\n if column and value is not None:\n operator = \"IN\" if isinstance(value, list) else \"==\"\n outgoing_clauses.append(\n {\"col\": column, \"op\": operator, \"val\": value}\n )\n\n normalized_filters.extend(outgoing_clauses)\n diagnostics.append(\n {\n \"filter_name\": display_name,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else None\n ),\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": deepcopy(outgoing_clauses),\n }\n )\n log.reason(\n \"Normalized effective preview filter for Superset query context\",\n payload={\n \"filter_name\": display_name,\n \"used_preserved_clauses\": used_preserved_clauses,\n \"outgoing_clauses\": outgoing_clauses,\n \"value_origin\": (\n normalized_payload.get(\"value_origin\")\n if isinstance(normalized_payload, dict)\n else \"heuristic_reconstruction\"\n ),\n },\n )\n\n return {\n \"filters\": normalized_filters,\n \"extra_form_data\": merged_extra_form_data,\n \"diagnostics\": diagnostics,\n }\n\n # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]\n" }, { "contract_id": "SupersetClientExtractCompiledSqlFromPreviewResponse", "contract_type": "Function", "file_path": "backend/src/core/superset_client/_datasets_preview_filters.py", - "start_line": 125, - "end_line": 197, + "start_line": 127, + "end_line": 199, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -42121,7 +42877,7 @@ "contract_type": "Module", "file_path": "backend/src/core/task_manager/manager.py", "start_line": 1, - "end_line": 828, + "end_line": 830, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -42343,14 +43099,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:TaskManagerModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: task, manager, lifecycle, execution, state\n# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.\n# @LAYER: Core\n# @PRE: Plugin loader and database sessions are initialized.\n# @POST: Orchestrates task execution and persistence.\n# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB.\n# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry]\n# @RELATION: [DEPENDS_ON] ->[PluginLoader]\n# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskContext]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @INVARIANT: Task IDs are unique.\n# @TEST_CONTRACT: TaskManagerRuntime -> {\n# required_fields: {plugin_loader: PluginLoader},\n# optional_fields: {},\n# invariants: [\"Must use belief_scope for logging\"]\n# }\n# @TEST_FIXTURE: valid_module -> {\"manager_initialized\": true}\n# @TEST_EDGE: missing_required_field -> {\"plugin_loader\": null}\n# @TEST_EDGE: empty_response -> {\"tasks\": []}\n# @TEST_EDGE: invalid_type -> {\"plugin_loader\": \"string_instead_of_object\"}\n# @TEST_EDGE: external_failure -> {\"db_unavailable\": true}\n# @TEST_INVARIANT: logger_compliance -> verifies: [valid_module]\n\n# [SECTION: IMPORTS]\nimport asyncio\nimport threading\nimport inspect\nfrom concurrent.futures import ThreadPoolExecutor\nfrom datetime import datetime, timezone\nfrom typing import Dict, Any, List, Optional\n\nfrom .models import Task, TaskStatus, LogEntry, LogFilter, LogStats\nfrom .persistence import TaskPersistenceService, TaskLogPersistenceService\nfrom .context import TaskContext\nfrom ..logger import logger, belief_scope, should_log_task_level\n# [/SECTION]\n\n\n# [DEF:TaskManager:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: task, manager, lifecycle, execution, state\n# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n# @LAYER: Core\n# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[PluginLoader]\n# @RELATION: [DEPENDS_ON] ->[TaskContext]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.\n# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.\n# @INVARIANT: Task IDs are unique within the registry.\n# @INVARIANT: Each task has exactly one status at any time.\n# @INVARIANT: Log entries are never deleted after being added to a task.\n# @SIDE_EFFECT: Spawns worker threads, flushes logs to database, and mutates task states.\n# @DATA_CONTRACT: Input[plugin_id, params] -> Output[Task]\nclass TaskManager:\n \"\"\"\n Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n \"\"\"\n\n # Log flush interval in seconds\n LOG_FLUSH_INTERVAL = 2.0\n\n # [DEF:TaskGraph:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.\n # @RELATION: [DEPENDS_ON] ->[Task]\n # @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n # @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.\n # @POST: Each live task id resolves to at most one active Task node and optional pause future.\n # @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.\n # @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]\n # @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.\n # [/DEF:TaskGraph:Block]\n\n # [DEF:EventBus:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.\n # @RELATION: [DEPENDS_ON] ->[LogEntry]\n # @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n # @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.\n # @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.\n # @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.\n # @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]\n # @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.\n # [/DEF:EventBus:Block]\n\n # [DEF:JobLifecycle:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[PluginLoader]\n # @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.\n # @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.\n # @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.\n # @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]\n # @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.\n # [/DEF:JobLifecycle:Block]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Initialize the TaskManager with dependencies.\n # @PRE: plugin_loader is initialized.\n # @POST: TaskManager is ready to accept tasks.\n # @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @PARAM: plugin_loader - The plugin loader instance.\n def __init__(self, plugin_loader):\n with belief_scope(\"TaskManager.__init__\"):\n logger.reason(\"Initializing task manager runtime services\")\n self.plugin_loader = plugin_loader\n self.tasks: Dict[str, Task] = {}\n self.subscribers: Dict[str, List[asyncio.Queue]] = {}\n self.executor = ThreadPoolExecutor(\n max_workers=5\n ) # For CPU-bound plugin execution\n self.persistence_service = TaskPersistenceService()\n self.log_persistence_service = TaskLogPersistenceService()\n\n # Log buffer: task_id -> List[LogEntry]\n self._log_buffer: Dict[str, List[LogEntry]] = {}\n self._log_buffer_lock = threading.Lock()\n\n # Flusher thread for batch writing logs\n self._flusher_stop_event = threading.Event()\n self._flusher_thread = threading.Thread(\n target=self._flusher_loop, daemon=True\n )\n self._flusher_thread.start()\n\n try:\n self.loop = asyncio.get_running_loop()\n except RuntimeError:\n self.loop = asyncio.get_event_loop()\n self.task_futures: Dict[str, asyncio.Future] = {}\n\n # Load persisted tasks on startup\n self.load_persisted_tasks()\n logger.reflect(\n \"Task manager runtime initialized\",\n extra={\"task_count\": len(self.tasks)},\n )\n\n # [/DEF:__init__:Function]\n\n # [DEF:_flusher_loop:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Background thread that periodically flushes log buffer to database.\n # @PRE: TaskManager is initialized.\n # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.\n # @RELATION: [CALLS] ->[TaskManager._flush_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flusher_loop(self):\n \"\"\"Background thread that flushes log buffer to database.\"\"\"\n while not self._flusher_stop_event.is_set():\n self._flush_logs()\n self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)\n\n # [/DEF:_flusher_loop:Function]\n\n # [DEF:_flush_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush all buffered logs to the database.\n # @PRE: None.\n # @POST: All buffered logs are written to task_logs table.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_logs(self):\n \"\"\"Flush all buffered logs to the database.\"\"\"\n with self._log_buffer_lock:\n task_ids = list(self._log_buffer.keys())\n\n for task_id in task_ids:\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n logger.debug(f\"Flushed {len(logs)} logs for task {task_id}\")\n except Exception as e:\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n # Re-add logs to buffer on failure\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].extend(logs)\n\n # [/DEF:_flush_logs:Function]\n\n # [DEF:_flush_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush logs for a specific task immediately.\n # @PRE: task_id exists.\n # @POST: Task's buffered logs are written to database.\n # @PARAM: task_id (str) - The task ID.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_task_logs(self, task_id: str):\n \"\"\"Flush logs for a specific task immediately.\"\"\"\n with belief_scope(\"_flush_task_logs\"):\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n except Exception as e:\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n\n # [/DEF:_flush_task_logs:Function]\n\n # [DEF:create_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Creates and queues a new task for execution.\n # @PRE: Plugin with plugin_id exists. Params are valid.\n # @POST: Task is created, added to registry, and scheduled for execution.\n # @PARAM: plugin_id (str) - The ID of the plugin to run.\n # @PARAM: params (Dict[str, Any]) - Parameters for the plugin.\n # @PARAM: user_id (Optional[str]) - ID of the user requesting the task.\n # @RETURN: Task - The created task instance.\n # @THROWS: ValueError if plugin not found or params invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n async def create_task(\n self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None\n ) -> Task:\n with belief_scope(\"TaskManager.create_task\", f\"plugin_id={plugin_id}\"):\n if not self.plugin_loader.has_plugin(plugin_id):\n logger.error(f\"Plugin with ID '{plugin_id}' not found.\")\n raise ValueError(f\"Plugin with ID '{plugin_id}' not found.\")\n\n self.plugin_loader.get_plugin(plugin_id)\n\n if not isinstance(params, dict):\n logger.error(\"Task parameters must be a dictionary.\")\n raise ValueError(\"Task parameters must be a dictionary.\")\n\n logger.reason(\"Creating task node and scheduling execution\")\n task = Task(plugin_id=plugin_id, params=params, user_id=user_id)\n self.tasks[task.id] = task\n self.persistence_service.persist_task(task)\n logger.info(f\"Task {task.id} created and scheduled for execution\")\n self.loop.create_task(\n self._run_task(task.id)\n ) # Schedule task for execution\n logger.reflect(\n \"Task creation persisted and execution scheduled\",\n extra={\"task_id\": task.id, \"plugin_id\": plugin_id},\n )\n return task\n\n # [/DEF:create_task:Function]\n\n # [DEF:_run_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Internal method to execute a task with TaskContext support.\n # @PRE: Task exists in registry.\n # @POST: Task is executed, status updated to SUCCESS or FAILED.\n # @PARAM: task_id (str) - The ID of the task to run.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def _run_task(self, task_id: str):\n with belief_scope(\"TaskManager._run_task\", f\"task_id={task_id}\"):\n task = self.tasks[task_id]\n plugin = self.plugin_loader.get_plugin(task.plugin_id)\n\n logger.reason(\n \"Transitioning task to running state\",\n extra={\"task_id\": task_id, \"plugin_id\": task.plugin_id},\n )\n logger.info(\n f\"Starting execution of task {task_id} for plugin '{plugin.name}'\"\n )\n task.status = TaskStatus.RUNNING\n task.started_at = datetime.utcnow()\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task started for plugin '{plugin.name}'\",\n source=\"system\",\n )\n\n try:\n # Prepare params and check if plugin supports new TaskContext\n params = {**task.params, \"_task_id\": task_id}\n\n # Check if plugin's execute method accepts 'context' parameter\n sig = inspect.signature(plugin.execute)\n accepts_context = \"context\" in sig.parameters\n\n if accepts_context:\n # Create TaskContext for new-style plugins\n context = TaskContext(\n task_id=task_id,\n add_log_fn=self._add_log,\n params=params,\n default_source=\"plugin\",\n background_tasks=None,\n )\n\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params, context=context)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor,\n lambda: plugin.execute(params, context=context),\n )\n else:\n # Backward compatibility: old-style plugins without context\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor, plugin.execute, params\n )\n\n logger.info(f\"Task {task_id} completed successfully\")\n task.status = TaskStatus.SUCCESS\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task completed successfully for plugin '{plugin.name}'\",\n source=\"system\",\n )\n except Exception as e:\n logger.error(f\"Task {task_id} failed: {e}\")\n task.status = TaskStatus.FAILED\n self._add_log(\n task_id,\n \"ERROR\",\n f\"Task failed: {e}\",\n source=\"system\",\n metadata={\"error_type\": type(e).__name__},\n )\n finally:\n task.finished_at = datetime.utcnow()\n # Flush any remaining buffered logs before persisting task\n self._flush_task_logs(task_id)\n self.persistence_service.persist_task(task)\n logger.info(\n f\"Task {task_id} execution finished with status: {task.status}\"\n )\n logger.reflect(\n \"Task lifecycle reached persisted terminal state\",\n extra={\"task_id\": task_id, \"status\": str(task.status)},\n )\n\n # [/DEF:_run_task:Function]\n\n # [DEF:resolve_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resumes a task that is awaiting mapping.\n # @PRE: Task exists and is in AWAITING_MAPPING state.\n # @POST: Task status updated to RUNNING, params updated, execution resumed.\n # @PARAM: task_id (str) - The ID of the task.\n # @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.\n # @THROWS: ValueError if task not found or not awaiting mapping.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):\n with belief_scope(\"TaskManager.resolve_task\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task or task.status != TaskStatus.AWAITING_MAPPING:\n raise ValueError(\"Task is not awaiting mapping.\")\n\n # Update task params with resolution\n task.params.update(resolution_params)\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(task_id, \"INFO\", \"Task resumed after mapping resolution.\")\n\n # Signal the future to continue\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resolve_task:Function]\n\n # [DEF:wait_for_resolution:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for a resolution signal.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_resolution(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_resolution\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n task.status = TaskStatus.AWAITING_MAPPING\n self.persistence_service.persist_task(task)\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_resolution:Function]\n\n # [DEF:wait_for_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for user input.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set via resume_task_with_password.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_input(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Status is already set to AWAITING_INPUT by await_input()\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_input:Function]\n\n # [DEF:get_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves a task by its ID.\n # @PRE: task_id is a string.\n # @POST: Returns Task object or None.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: Optional[Task] - The task or None.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_task(self, task_id: str) -> Optional[Task]:\n with belief_scope(\"TaskManager.get_task\", f\"task_id={task_id}\"):\n return self.tasks.get(task_id)\n\n # [/DEF:get_task:Function]\n\n # [DEF:get_all_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves all registered tasks.\n # @PRE: None.\n # @POST: Returns list of all Task objects.\n # @RETURN: List[Task] - All tasks.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_all_tasks(self) -> List[Task]:\n with belief_scope(\"TaskManager.get_all_tasks\"):\n return list(self.tasks.values())\n\n # [/DEF:get_all_tasks:Function]\n\n # [DEF:get_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves tasks with pagination and optional status filter.\n # @PRE: limit and offset are non-negative integers.\n # @POST: Returns a list of tasks sorted by start_time descending.\n # @PARAM: limit (int) - Maximum number of tasks to return.\n # @PARAM: offset (int) - Number of tasks to skip.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: List[Task] - List of tasks matching criteria.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_tasks(\n self,\n limit: int = 10,\n offset: int = 0,\n status: Optional[TaskStatus] = None,\n plugin_ids: Optional[List[str]] = None,\n completed_only: bool = False,\n ) -> List[Task]:\n with belief_scope(\"TaskManager.get_tasks\"):\n tasks = list(self.tasks.values())\n if status:\n tasks = [t for t in tasks if t.status == status]\n if plugin_ids:\n plugin_id_set = set(plugin_ids)\n tasks = [t for t in tasks if t.plugin_id in plugin_id_set]\n if completed_only:\n tasks = [\n t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]\n ]\n\n # Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.\n def sort_key(task: Task) -> float:\n started_at = task.started_at\n if started_at is None:\n return float(\"-inf\")\n if not isinstance(started_at, datetime):\n return float(\"-inf\")\n if started_at.tzinfo is None:\n return started_at.replace(tzinfo=timezone.utc).timestamp()\n return started_at.timestamp()\n\n tasks.sort(key=sort_key, reverse=True)\n return tasks[offset : offset + limit]\n\n # [/DEF:get_tasks:Function]\n\n # [DEF:get_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).\n # @PRE: task_id is a string.\n # @POST: Returns list of LogEntry or TaskLog objects.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.\n # @RETURN: List[LogEntry] - List of log entries.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_logs(\n self, task_id: str, log_filter: Optional[LogFilter] = None\n ) -> List[LogEntry]:\n with belief_scope(\"TaskManager.get_task_logs\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n\n # For completed tasks, fetch from persistence\n if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:\n if log_filter is None:\n log_filter = LogFilter()\n task_logs = self.log_persistence_service.get_logs(task_id, log_filter)\n # Convert TaskLog to LogEntry for backward compatibility\n return [\n LogEntry(\n timestamp=log.timestamp,\n level=log.level,\n message=log.message,\n source=log.source,\n metadata=log.metadata,\n )\n for log in task_logs\n ]\n\n # For running/pending tasks, return from memory\n return task.logs if task else []\n\n # [/DEF:get_task_logs:Function]\n\n # [DEF:get_task_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\"TaskManager.get_task_log_stats\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_log_stats(task_id)\n\n # [/DEF:get_task_log_stats:Function]\n\n # [DEF:get_task_log_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_sources(self, task_id: str) -> List[str]:\n with belief_scope(\"TaskManager.get_task_log_sources\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_sources(task_id)\n\n # [/DEF:get_task_log_sources:Function]\n\n # [DEF:_add_log:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.\n # @PRE: Task exists.\n # @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: level (str) - Log level.\n # @PARAM: message (str) - Log message.\n # @PARAM: source (str) - Source component (default: \"system\").\n # @PARAM: metadata (Optional[Dict]) - Additional structured data.\n # @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).\n # @RELATION: [CALLS] ->[should_log_task_level]\n # @RELATION: [DISPATCHES] ->[EventBus]\n def _add_log(\n self,\n task_id: str,\n level: str,\n message: str,\n source: str = \"system\",\n metadata: Optional[Dict[str, Any]] = None,\n context: Optional[Dict[str, Any]] = None,\n ):\n with belief_scope(\"TaskManager._add_log\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Filter logs based on task_log_level configuration\n if not should_log_task_level(level):\n return\n\n # Create log entry with new fields\n log_entry = LogEntry(\n level=level,\n message=message,\n source=source,\n metadata=metadata,\n context=context, # Keep for backward compatibility\n )\n\n # Add to in-memory logs (for backward compatibility with legacy JSON field)\n task.logs.append(log_entry)\n\n # Add to buffer for batch persistence\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].append(log_entry)\n\n # Notify subscribers (for real-time WebSocket updates)\n if task_id in self.subscribers:\n for queue in self.subscribers[task_id]:\n self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)\n\n # [/DEF:_add_log:Function]\n\n # [DEF:subscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Subscribes to real-time logs for a task.\n # @PRE: task_id is a string.\n # @POST: Returns an asyncio.Queue for log entries.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: asyncio.Queue - Queue for log entries.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def subscribe_logs(self, task_id: str) -> asyncio.Queue:\n with belief_scope(\"TaskManager.subscribe_logs\", f\"task_id={task_id}\"):\n queue = asyncio.Queue()\n if task_id not in self.subscribers:\n self.subscribers[task_id] = []\n self.subscribers[task_id].append(queue)\n return queue\n\n # [/DEF:subscribe_logs:Function]\n\n # [DEF:unsubscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Unsubscribes from real-time logs for a task.\n # @PRE: task_id is a string, queue is asyncio.Queue.\n # @POST: Queue removed from subscribers.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: queue (asyncio.Queue) - Queue to remove.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):\n with belief_scope(\"TaskManager.unsubscribe_logs\", f\"task_id={task_id}\"):\n if task_id in self.subscribers:\n if queue in self.subscribers[task_id]:\n self.subscribers[task_id].remove(queue)\n if not self.subscribers[task_id]:\n del self.subscribers[task_id]\n\n # [/DEF:unsubscribe_logs:Function]\n\n # [DEF:load_persisted_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Load persisted tasks using persistence service.\n # @PRE: None.\n # @POST: Persisted tasks loaded into self.tasks.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def load_persisted_tasks(self) -> None:\n with belief_scope(\"TaskManager.load_persisted_tasks\"):\n loaded_tasks = self.persistence_service.load_tasks(limit=100)\n for task in loaded_tasks:\n if task.id not in self.tasks:\n self.tasks[task.id] = task\n\n # [/DEF:load_persisted_tasks:Function]\n\n # [DEF:await_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Transition a task to AWAITING_INPUT state with input request.\n # @PRE: Task exists and is in RUNNING state.\n # @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: input_request (Dict) - Details about required input.\n # @THROWS: ValueError if task not found or not RUNNING.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:\n with belief_scope(\"TaskManager.await_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.RUNNING:\n raise ValueError(\n f\"Task {task_id} is not RUNNING (current: {task.status})\"\n )\n\n task.status = TaskStatus.AWAITING_INPUT\n task.input_required = True\n task.input_request = input_request\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task paused for user input\",\n metadata={\"input_request\": input_request},\n )\n\n # [/DEF:await_input:Function]\n\n # [DEF:resume_task_with_password:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resume a task that is awaiting input with provided passwords.\n # @PRE: Task exists and is in AWAITING_INPUT state.\n # @POST: Task status changed to RUNNING, passwords injected, task resumed.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.\n # @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def resume_task_with_password(\n self, task_id: str, passwords: Dict[str, str]\n ) -> None:\n with belief_scope(\n \"TaskManager.resume_task_with_password\", f\"task_id={task_id}\"\n ):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.AWAITING_INPUT:\n raise ValueError(\n f\"Task {task_id} is not AWAITING_INPUT (current: {task.status})\"\n )\n\n if not isinstance(passwords, dict) or not passwords:\n raise ValueError(\"Passwords must be a non-empty dictionary\")\n\n task.params[\"passwords\"] = passwords\n task.input_required = False\n task.input_request = None\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task resumed with passwords\",\n metadata={\"databases\": list(passwords.keys())},\n )\n\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resume_task_with_password:Function]\n\n # [DEF:clear_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Clears tasks based on status filter (also deletes associated logs).\n # @PRE: status is Optional[TaskStatus].\n # @POST: Tasks matching filter (or all non-active) cleared from registry and database.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: int - Number of tasks cleared.\n # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:\n with belief_scope(\"TaskManager.clear_tasks\"):\n tasks_to_remove = []\n for task_id, task in list(self.tasks.items()):\n # If status is provided, match it.\n # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)\n # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.\n # RUNNING is active execution.\n\n should_remove = False\n if status:\n if task.status == status:\n should_remove = True\n else:\n # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)\n if task.status not in [\n TaskStatus.RUNNING,\n TaskStatus.AWAITING_INPUT,\n TaskStatus.AWAITING_MAPPING,\n ]:\n should_remove = True\n\n if should_remove:\n tasks_to_remove.append(task_id)\n\n for tid in tasks_to_remove:\n # Cancel future if exists (e.g. for AWAITING_INPUT/MAPPING)\n if tid in self.task_futures:\n self.task_futures[tid].cancel()\n del self.task_futures[tid]\n\n del self.tasks[tid]\n\n # Remove from persistence (task_records and task_logs via CASCADE)\n self.persistence_service.delete_tasks(tasks_to_remove)\n\n # Also explicitly delete logs (in case CASCADE is not set up)\n if tasks_to_remove:\n self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)\n\n logger.info(f\"Cleared {len(tasks_to_remove)} tasks.\")\n return len(tasks_to_remove)\n\n # [/DEF:clear_tasks:Function]\n\n\n# [/DEF:TaskManager:Class]\n# [/DEF:TaskManagerModule:Module]\n" + "body": "# [DEF:TaskManagerModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: task, manager, lifecycle, execution, state\n# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.\n# @LAYER: Core\n# @PRE: Plugin loader and database sessions are initialized.\n# @POST: Orchestrates task execution and persistence.\n# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB.\n# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry]\n# @RELATION: [DEPENDS_ON] ->[PluginLoader]\n# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskContext]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @INVARIANT: Task IDs are unique.\n# @TEST_CONTRACT: TaskManagerRuntime -> {\n# required_fields: {plugin_loader: PluginLoader},\n# optional_fields: {},\n# invariants: [\"Must use belief_scope for logging\"]\n# }\n# @TEST_FIXTURE: valid_module -> {\"manager_initialized\": true}\n# @TEST_EDGE: missing_required_field -> {\"plugin_loader\": null}\n# @TEST_EDGE: empty_response -> {\"tasks\": []}\n# @TEST_EDGE: invalid_type -> {\"plugin_loader\": \"string_instead_of_object\"}\n# @TEST_EDGE: external_failure -> {\"db_unavailable\": true}\n# @TEST_INVARIANT: logger_compliance -> verifies: [valid_module]\n\n# [SECTION: IMPORTS]\nimport asyncio\nimport threading\nimport inspect\nfrom concurrent.futures import ThreadPoolExecutor\nfrom datetime import datetime, timezone\nfrom typing import Dict, Any, List, Optional\n\nfrom .models import Task, TaskStatus, LogEntry, LogFilter, LogStats\nfrom .persistence import TaskPersistenceService, TaskLogPersistenceService\nfrom .context import TaskContext\nfrom ..logger import logger, belief_scope, should_log_task_level\nfrom ..cot_logger import MarkerLogger, get_trace_id, set_trace_id\n\nlog = MarkerLogger(\"TaskManager\")\n# [/SECTION]\n\n\n# [DEF:TaskManager:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: task, manager, lifecycle, execution, state\n# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n# @LAYER: Core\n# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[PluginLoader]\n# @RELATION: [DEPENDS_ON] ->[TaskContext]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.\n# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.\n# @INVARIANT: Task IDs are unique within the registry.\n# @INVARIANT: Each task has exactly one status at any time.\n# @INVARIANT: Log entries are never deleted after being added to a task.\n# @SIDE_EFFECT: Spawns worker threads, flushes logs to database, and mutates task states.\n# @DATA_CONTRACT: Input[plugin_id, params] -> Output[Task]\nclass TaskManager:\n \"\"\"\n Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n \"\"\"\n\n # Log flush interval in seconds\n LOG_FLUSH_INTERVAL = 2.0\n\n # [DEF:TaskGraph:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.\n # @RELATION: [DEPENDS_ON] ->[Task]\n # @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n # @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.\n # @POST: Each live task id resolves to at most one active Task node and optional pause future.\n # @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.\n # @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]\n # @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.\n # [/DEF:TaskGraph:Block]\n\n # [DEF:EventBus:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.\n # @RELATION: [DEPENDS_ON] ->[LogEntry]\n # @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n # @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.\n # @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.\n # @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.\n # @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]\n # @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.\n # [/DEF:EventBus:Block]\n\n # [DEF:JobLifecycle:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[PluginLoader]\n # @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.\n # @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.\n # @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.\n # @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]\n # @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.\n # [/DEF:JobLifecycle:Block]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Initialize the TaskManager with dependencies.\n # @PRE: plugin_loader is initialized.\n # @POST: TaskManager is ready to accept tasks.\n # @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @PARAM: plugin_loader - The plugin loader instance.\n def __init__(self, plugin_loader):\n with belief_scope(\"TaskManager.__init__\"):\n log.reason(\"Initializing task manager runtime services\")\n self.plugin_loader = plugin_loader\n self.tasks: Dict[str, Task] = {}\n self.subscribers: Dict[str, List[asyncio.Queue]] = {}\n self.executor = ThreadPoolExecutor(\n max_workers=5\n ) # For CPU-bound plugin execution\n self.persistence_service = TaskPersistenceService()\n self.log_persistence_service = TaskLogPersistenceService()\n\n # Log buffer: task_id -> List[LogEntry]\n self._log_buffer: Dict[str, List[LogEntry]] = {}\n self._log_buffer_lock = threading.Lock()\n\n # Flusher thread for batch writing logs\n self._flusher_stop_event = threading.Event()\n self._flusher_thread = threading.Thread(\n target=self._flusher_loop, daemon=True\n )\n self._flusher_thread.start()\n\n try:\n self.loop = asyncio.get_running_loop()\n except RuntimeError:\n self.loop = asyncio.get_event_loop()\n self.task_futures: Dict[str, asyncio.Future] = {}\n\n # Load persisted tasks on startup\n self.load_persisted_tasks()\n log.reflect(\n \"Task manager runtime initialized\",\n payload={\"task_count\": len(self.tasks)},\n )\n\n # [/DEF:__init__:Function]\n\n # [DEF:_flusher_loop:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Background thread that periodically flushes log buffer to database.\n # @PRE: TaskManager is initialized.\n # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.\n # @RELATION: [CALLS] ->[TaskManager._flush_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flusher_loop(self):\n \"\"\"Background thread that flushes log buffer to database.\"\"\"\n while not self._flusher_stop_event.is_set():\n self._flush_logs()\n self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)\n\n # [/DEF:_flusher_loop:Function]\n\n # [DEF:_flush_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush all buffered logs to the database.\n # @PRE: None.\n # @POST: All buffered logs are written to task_logs table.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_logs(self):\n \"\"\"Flush all buffered logs to the database.\"\"\"\n with self._log_buffer_lock:\n task_ids = list(self._log_buffer.keys())\n\n for task_id in task_ids:\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n log.reflect(\"Flushed logs to database\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n log.explore(\"Failed to flush logs to database\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n # Re-add logs to buffer on failure\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].extend(logs)\n\n # [/DEF:_flush_logs:Function]\n\n # [DEF:_flush_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush logs for a specific task immediately.\n # @PRE: task_id exists.\n # @POST: Task's buffered logs are written to database.\n # @PARAM: task_id (str) - The task ID.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_task_logs(self, task_id: str):\n \"\"\"Flush logs for a specific task immediately.\"\"\"\n with belief_scope(\"_flush_task_logs\"):\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n log.reflect(\"Flushed task logs immediately\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n log.explore(\"Failed to flush task logs immediately\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n\n # [/DEF:_flush_task_logs:Function]\n\n # [DEF:create_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Creates and queues a new task for execution.\n # @PRE: Plugin with plugin_id exists. Params are valid.\n # @POST: Task is created, added to registry, and scheduled for execution.\n # @PARAM: plugin_id (str) - The ID of the plugin to run.\n # @PARAM: params (Dict[str, Any]) - Parameters for the plugin.\n # @PARAM: user_id (Optional[str]) - ID of the user requesting the task.\n # @RETURN: Task - The created task instance.\n # @THROWS: ValueError if plugin not found or params invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n async def create_task(\n self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None\n ) -> Task:\n with belief_scope(\"TaskManager.create_task\", f\"plugin_id={plugin_id}\"):\n if not self.plugin_loader.has_plugin(plugin_id):\n log.explore(\"Plugin not found\", error=f\"Plugin with ID '{plugin_id}' not found.\")\n raise ValueError(f\"Plugin with ID '{plugin_id}' not found.\")\n\n self.plugin_loader.get_plugin(plugin_id)\n\n if not isinstance(params, dict):\n log.explore(\"Invalid task params type\", error=\"Task parameters must be a dictionary.\")\n raise ValueError(\"Task parameters must be a dictionary.\")\n\n log.reason(\"Creating task node and scheduling execution\", payload={\"plugin_id\": plugin_id})\n task = Task(plugin_id=plugin_id, params=params, user_id=user_id)\n self.tasks[task.id] = task\n self.persistence_service.persist_task(task)\n trace_id = get_trace_id()\n self.loop.create_task(\n self._run_task(task.id, trace_id=trace_id)\n ) # Schedule task for execution\n log.reflect(\n \"Task creation persisted and execution scheduled\",\n payload={\"task_id\": task.id, \"plugin_id\": plugin_id},\n )\n return task\n\n # [/DEF:create_task:Function]\n\n # [DEF:_run_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Internal method to execute a task with TaskContext support.\n # @PRE: Task exists in registry.\n # @POST: Task is executed, status updated to SUCCESS or FAILED.\n # @PARAM: task_id (str) - The ID of the task to run.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def _run_task(self, task_id: str, trace_id: str = None):\n if trace_id:\n set_trace_id(trace_id)\n with belief_scope(\"TaskManager._run_task\", f\"task_id={task_id}\"):\n task = self.tasks[task_id]\n plugin = self.plugin_loader.get_plugin(task.plugin_id)\n\n log.reason(\n \"Transitioning task to running state\",\n payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id},\n )\n task.status = TaskStatus.RUNNING\n task.started_at = datetime.utcnow()\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task started for plugin '{plugin.name}'\",\n source=\"system\",\n )\n\n try:\n # Prepare params and check if plugin supports new TaskContext\n params = {**task.params, \"_task_id\": task_id}\n\n # Check if plugin's execute method accepts 'context' parameter\n sig = inspect.signature(plugin.execute)\n accepts_context = \"context\" in sig.parameters\n\n if accepts_context:\n # Create TaskContext for new-style plugins\n context = TaskContext(\n task_id=task_id,\n add_log_fn=self._add_log,\n params=params,\n default_source=\"plugin\",\n background_tasks=None,\n )\n\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params, context=context)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor,\n lambda: plugin.execute(params, context=context),\n )\n else:\n # Backward compatibility: old-style plugins without context\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor, plugin.execute, params\n )\n\n task.status = TaskStatus.SUCCESS\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task completed successfully for plugin '{plugin.name}'\",\n source=\"system\",\n )\n log.reflect(\"Task completed successfully\", payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id})\n except Exception as e:\n task.status = TaskStatus.FAILED\n self._add_log(\n task_id,\n \"ERROR\",\n f\"Task failed: {e}\",\n source=\"system\",\n metadata={\"error_type\": type(e).__name__},\n )\n log.explore(\"Task failed during execution\", error=str(e), payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id})\n finally:\n task.finished_at = datetime.utcnow()\n # Flush any remaining buffered logs before persisting task\n self._flush_task_logs(task_id)\n self.persistence_service.persist_task(task)\n log.reflect(\n \"Task execution finished\",\n payload={\"task_id\": task_id, \"status\": str(task.status)},\n )\n\n # [/DEF:_run_task:Function]\n\n # [DEF:resolve_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resumes a task that is awaiting mapping.\n # @PRE: Task exists and is in AWAITING_MAPPING state.\n # @POST: Task status updated to RUNNING, params updated, execution resumed.\n # @PARAM: task_id (str) - The ID of the task.\n # @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.\n # @THROWS: ValueError if task not found or not awaiting mapping.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):\n with belief_scope(\"TaskManager.resolve_task\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task or task.status != TaskStatus.AWAITING_MAPPING:\n raise ValueError(\"Task is not awaiting mapping.\")\n\n # Update task params with resolution\n task.params.update(resolution_params)\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(task_id, \"INFO\", \"Task resumed after mapping resolution.\")\n\n # Signal the future to continue\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resolve_task:Function]\n\n # [DEF:wait_for_resolution:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for a resolution signal.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_resolution(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_resolution\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n task.status = TaskStatus.AWAITING_MAPPING\n self.persistence_service.persist_task(task)\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_resolution:Function]\n\n # [DEF:wait_for_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for user input.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set via resume_task_with_password.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_input(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Status is already set to AWAITING_INPUT by await_input()\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_input:Function]\n\n # [DEF:get_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves a task by its ID.\n # @PRE: task_id is a string.\n # @POST: Returns Task object or None.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: Optional[Task] - The task or None.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_task(self, task_id: str) -> Optional[Task]:\n with belief_scope(\"TaskManager.get_task\", f\"task_id={task_id}\"):\n return self.tasks.get(task_id)\n\n # [/DEF:get_task:Function]\n\n # [DEF:get_all_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves all registered tasks.\n # @PRE: None.\n # @POST: Returns list of all Task objects.\n # @RETURN: List[Task] - All tasks.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_all_tasks(self) -> List[Task]:\n with belief_scope(\"TaskManager.get_all_tasks\"):\n return list(self.tasks.values())\n\n # [/DEF:get_all_tasks:Function]\n\n # [DEF:get_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves tasks with pagination and optional status filter.\n # @PRE: limit and offset are non-negative integers.\n # @POST: Returns a list of tasks sorted by start_time descending.\n # @PARAM: limit (int) - Maximum number of tasks to return.\n # @PARAM: offset (int) - Number of tasks to skip.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: List[Task] - List of tasks matching criteria.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_tasks(\n self,\n limit: int = 10,\n offset: int = 0,\n status: Optional[TaskStatus] = None,\n plugin_ids: Optional[List[str]] = None,\n completed_only: bool = False,\n ) -> List[Task]:\n with belief_scope(\"TaskManager.get_tasks\"):\n tasks = list(self.tasks.values())\n if status:\n tasks = [t for t in tasks if t.status == status]\n if plugin_ids:\n plugin_id_set = set(plugin_ids)\n tasks = [t for t in tasks if t.plugin_id in plugin_id_set]\n if completed_only:\n tasks = [\n t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]\n ]\n\n # Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.\n def sort_key(task: Task) -> float:\n started_at = task.started_at\n if started_at is None:\n return float(\"-inf\")\n if not isinstance(started_at, datetime):\n return float(\"-inf\")\n if started_at.tzinfo is None:\n return started_at.replace(tzinfo=timezone.utc).timestamp()\n return started_at.timestamp()\n\n tasks.sort(key=sort_key, reverse=True)\n return tasks[offset : offset + limit]\n\n # [/DEF:get_tasks:Function]\n\n # [DEF:get_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).\n # @PRE: task_id is a string.\n # @POST: Returns list of LogEntry or TaskLog objects.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.\n # @RETURN: List[LogEntry] - List of log entries.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_logs(\n self, task_id: str, log_filter: Optional[LogFilter] = None\n ) -> List[LogEntry]:\n with belief_scope(\"TaskManager.get_task_logs\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n\n # For completed tasks, fetch from persistence\n if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:\n if log_filter is None:\n log_filter = LogFilter()\n task_logs = self.log_persistence_service.get_logs(task_id, log_filter)\n # Convert TaskLog to LogEntry for backward compatibility\n return [\n LogEntry(\n timestamp=log.timestamp,\n level=log.level,\n message=log.message,\n source=log.source,\n metadata=log.metadata,\n )\n for log in task_logs\n ]\n\n # For running/pending tasks, return from memory\n return task.logs if task else []\n\n # [/DEF:get_task_logs:Function]\n\n # [DEF:get_task_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\"TaskManager.get_task_log_stats\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_log_stats(task_id)\n\n # [/DEF:get_task_log_stats:Function]\n\n # [DEF:get_task_log_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_sources(self, task_id: str) -> List[str]:\n with belief_scope(\"TaskManager.get_task_log_sources\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_sources(task_id)\n\n # [/DEF:get_task_log_sources:Function]\n\n # [DEF:_add_log:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.\n # @PRE: Task exists.\n # @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: level (str) - Log level.\n # @PARAM: message (str) - Log message.\n # @PARAM: source (str) - Source component (default: \"system\").\n # @PARAM: metadata (Optional[Dict]) - Additional structured data.\n # @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).\n # @RELATION: [CALLS] ->[should_log_task_level]\n # @RELATION: [DISPATCHES] ->[EventBus]\n def _add_log(\n self,\n task_id: str,\n level: str,\n message: str,\n source: str = \"system\",\n metadata: Optional[Dict[str, Any]] = None,\n context: Optional[Dict[str, Any]] = None,\n ):\n with belief_scope(\"TaskManager._add_log\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Filter logs based on task_log_level configuration\n if not should_log_task_level(level):\n return\n\n # Create log entry with new fields\n log_entry = LogEntry(\n level=level,\n message=message,\n source=source,\n metadata=metadata,\n context=context, # Keep for backward compatibility\n )\n\n # Add to in-memory logs (for backward compatibility with legacy JSON field)\n task.logs.append(log_entry)\n\n # Add to buffer for batch persistence\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].append(log_entry)\n\n # Notify subscribers (for real-time WebSocket updates)\n if task_id in self.subscribers:\n for queue in self.subscribers[task_id]:\n self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)\n\n # [/DEF:_add_log:Function]\n\n # [DEF:subscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Subscribes to real-time logs for a task.\n # @PRE: task_id is a string.\n # @POST: Returns an asyncio.Queue for log entries.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: asyncio.Queue - Queue for log entries.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def subscribe_logs(self, task_id: str) -> asyncio.Queue:\n with belief_scope(\"TaskManager.subscribe_logs\", f\"task_id={task_id}\"):\n queue = asyncio.Queue()\n if task_id not in self.subscribers:\n self.subscribers[task_id] = []\n self.subscribers[task_id].append(queue)\n return queue\n\n # [/DEF:subscribe_logs:Function]\n\n # [DEF:unsubscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Unsubscribes from real-time logs for a task.\n # @PRE: task_id is a string, queue is asyncio.Queue.\n # @POST: Queue removed from subscribers.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: queue (asyncio.Queue) - Queue to remove.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):\n with belief_scope(\"TaskManager.unsubscribe_logs\", f\"task_id={task_id}\"):\n if task_id in self.subscribers:\n if queue in self.subscribers[task_id]:\n self.subscribers[task_id].remove(queue)\n if not self.subscribers[task_id]:\n del self.subscribers[task_id]\n\n # [/DEF:unsubscribe_logs:Function]\n\n # [DEF:load_persisted_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Load persisted tasks using persistence service.\n # @PRE: None.\n # @POST: Persisted tasks loaded into self.tasks.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def load_persisted_tasks(self) -> None:\n with belief_scope(\"TaskManager.load_persisted_tasks\"):\n loaded_tasks = self.persistence_service.load_tasks(limit=100)\n for task in loaded_tasks:\n if task.id not in self.tasks:\n self.tasks[task.id] = task\n\n # [/DEF:load_persisted_tasks:Function]\n\n # [DEF:await_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Transition a task to AWAITING_INPUT state with input request.\n # @PRE: Task exists and is in RUNNING state.\n # @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: input_request (Dict) - Details about required input.\n # @THROWS: ValueError if task not found or not RUNNING.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:\n with belief_scope(\"TaskManager.await_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.RUNNING:\n raise ValueError(\n f\"Task {task_id} is not RUNNING (current: {task.status})\"\n )\n\n task.status = TaskStatus.AWAITING_INPUT\n task.input_required = True\n task.input_request = input_request\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task paused for user input\",\n metadata={\"input_request\": input_request},\n )\n\n # [/DEF:await_input:Function]\n\n # [DEF:resume_task_with_password:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resume a task that is awaiting input with provided passwords.\n # @PRE: Task exists and is in AWAITING_INPUT state.\n # @POST: Task status changed to RUNNING, passwords injected, task resumed.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.\n # @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def resume_task_with_password(\n self, task_id: str, passwords: Dict[str, str]\n ) -> None:\n with belief_scope(\n \"TaskManager.resume_task_with_password\", f\"task_id={task_id}\"\n ):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.AWAITING_INPUT:\n raise ValueError(\n f\"Task {task_id} is not AWAITING_INPUT (current: {task.status})\"\n )\n\n if not isinstance(passwords, dict) or not passwords:\n raise ValueError(\"Passwords must be a non-empty dictionary\")\n\n task.params[\"passwords\"] = passwords\n task.input_required = False\n task.input_request = None\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task resumed with passwords\",\n metadata={\"databases\": list(passwords.keys())},\n )\n\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resume_task_with_password:Function]\n\n # [DEF:clear_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Clears tasks based on status filter (also deletes associated logs).\n # @PRE: status is Optional[TaskStatus].\n # @POST: Tasks matching filter (or all non-active) cleared from registry and database.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: int - Number of tasks cleared.\n # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:\n with belief_scope(\"TaskManager.clear_tasks\"):\n tasks_to_remove = []\n for task_id, task in list(self.tasks.items()):\n # If status is provided, match it.\n # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)\n # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.\n # RUNNING is active execution.\n\n should_remove = False\n if status:\n if task.status == status:\n should_remove = True\n else:\n # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)\n if task.status not in [\n TaskStatus.RUNNING,\n TaskStatus.AWAITING_INPUT,\n TaskStatus.AWAITING_MAPPING,\n ]:\n should_remove = True\n\n if should_remove:\n tasks_to_remove.append(task_id)\n\n for tid in tasks_to_remove:\n # Cancel future if exists (e.g. for AWAITING_INPUT/MAPPING)\n if tid in self.task_futures:\n self.task_futures[tid].cancel()\n del self.task_futures[tid]\n\n del self.tasks[tid]\n\n # Remove from persistence (task_records and task_logs via CASCADE)\n self.persistence_service.delete_tasks(tasks_to_remove)\n\n # Also explicitly delete logs (in case CASCADE is not set up)\n if tasks_to_remove:\n self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)\n\n log.reflect(\"Tasks cleared from registry and database\", payload={\"count\": len(tasks_to_remove)})\n return len(tasks_to_remove)\n\n # [/DEF:clear_tasks:Function]\n\n\n# [/DEF:TaskManager:Class]\n# [/DEF:TaskManagerModule:Module]\n" }, { "contract_id": "TaskManager", "contract_type": "Class", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 45, - "end_line": 827, + "start_line": 48, + "end_line": 829, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -42590,14 +43346,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:TaskManager:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: task, manager, lifecycle, execution, state\n# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n# @LAYER: Core\n# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[PluginLoader]\n# @RELATION: [DEPENDS_ON] ->[TaskContext]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.\n# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.\n# @INVARIANT: Task IDs are unique within the registry.\n# @INVARIANT: Each task has exactly one status at any time.\n# @INVARIANT: Log entries are never deleted after being added to a task.\n# @SIDE_EFFECT: Spawns worker threads, flushes logs to database, and mutates task states.\n# @DATA_CONTRACT: Input[plugin_id, params] -> Output[Task]\nclass TaskManager:\n \"\"\"\n Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n \"\"\"\n\n # Log flush interval in seconds\n LOG_FLUSH_INTERVAL = 2.0\n\n # [DEF:TaskGraph:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.\n # @RELATION: [DEPENDS_ON] ->[Task]\n # @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n # @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.\n # @POST: Each live task id resolves to at most one active Task node and optional pause future.\n # @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.\n # @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]\n # @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.\n # [/DEF:TaskGraph:Block]\n\n # [DEF:EventBus:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.\n # @RELATION: [DEPENDS_ON] ->[LogEntry]\n # @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n # @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.\n # @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.\n # @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.\n # @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]\n # @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.\n # [/DEF:EventBus:Block]\n\n # [DEF:JobLifecycle:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[PluginLoader]\n # @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.\n # @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.\n # @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.\n # @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]\n # @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.\n # [/DEF:JobLifecycle:Block]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Initialize the TaskManager with dependencies.\n # @PRE: plugin_loader is initialized.\n # @POST: TaskManager is ready to accept tasks.\n # @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @PARAM: plugin_loader - The plugin loader instance.\n def __init__(self, plugin_loader):\n with belief_scope(\"TaskManager.__init__\"):\n logger.reason(\"Initializing task manager runtime services\")\n self.plugin_loader = plugin_loader\n self.tasks: Dict[str, Task] = {}\n self.subscribers: Dict[str, List[asyncio.Queue]] = {}\n self.executor = ThreadPoolExecutor(\n max_workers=5\n ) # For CPU-bound plugin execution\n self.persistence_service = TaskPersistenceService()\n self.log_persistence_service = TaskLogPersistenceService()\n\n # Log buffer: task_id -> List[LogEntry]\n self._log_buffer: Dict[str, List[LogEntry]] = {}\n self._log_buffer_lock = threading.Lock()\n\n # Flusher thread for batch writing logs\n self._flusher_stop_event = threading.Event()\n self._flusher_thread = threading.Thread(\n target=self._flusher_loop, daemon=True\n )\n self._flusher_thread.start()\n\n try:\n self.loop = asyncio.get_running_loop()\n except RuntimeError:\n self.loop = asyncio.get_event_loop()\n self.task_futures: Dict[str, asyncio.Future] = {}\n\n # Load persisted tasks on startup\n self.load_persisted_tasks()\n logger.reflect(\n \"Task manager runtime initialized\",\n extra={\"task_count\": len(self.tasks)},\n )\n\n # [/DEF:__init__:Function]\n\n # [DEF:_flusher_loop:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Background thread that periodically flushes log buffer to database.\n # @PRE: TaskManager is initialized.\n # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.\n # @RELATION: [CALLS] ->[TaskManager._flush_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flusher_loop(self):\n \"\"\"Background thread that flushes log buffer to database.\"\"\"\n while not self._flusher_stop_event.is_set():\n self._flush_logs()\n self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)\n\n # [/DEF:_flusher_loop:Function]\n\n # [DEF:_flush_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush all buffered logs to the database.\n # @PRE: None.\n # @POST: All buffered logs are written to task_logs table.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_logs(self):\n \"\"\"Flush all buffered logs to the database.\"\"\"\n with self._log_buffer_lock:\n task_ids = list(self._log_buffer.keys())\n\n for task_id in task_ids:\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n logger.debug(f\"Flushed {len(logs)} logs for task {task_id}\")\n except Exception as e:\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n # Re-add logs to buffer on failure\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].extend(logs)\n\n # [/DEF:_flush_logs:Function]\n\n # [DEF:_flush_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush logs for a specific task immediately.\n # @PRE: task_id exists.\n # @POST: Task's buffered logs are written to database.\n # @PARAM: task_id (str) - The task ID.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_task_logs(self, task_id: str):\n \"\"\"Flush logs for a specific task immediately.\"\"\"\n with belief_scope(\"_flush_task_logs\"):\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n except Exception as e:\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n\n # [/DEF:_flush_task_logs:Function]\n\n # [DEF:create_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Creates and queues a new task for execution.\n # @PRE: Plugin with plugin_id exists. Params are valid.\n # @POST: Task is created, added to registry, and scheduled for execution.\n # @PARAM: plugin_id (str) - The ID of the plugin to run.\n # @PARAM: params (Dict[str, Any]) - Parameters for the plugin.\n # @PARAM: user_id (Optional[str]) - ID of the user requesting the task.\n # @RETURN: Task - The created task instance.\n # @THROWS: ValueError if plugin not found or params invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n async def create_task(\n self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None\n ) -> Task:\n with belief_scope(\"TaskManager.create_task\", f\"plugin_id={plugin_id}\"):\n if not self.plugin_loader.has_plugin(plugin_id):\n logger.error(f\"Plugin with ID '{plugin_id}' not found.\")\n raise ValueError(f\"Plugin with ID '{plugin_id}' not found.\")\n\n self.plugin_loader.get_plugin(plugin_id)\n\n if not isinstance(params, dict):\n logger.error(\"Task parameters must be a dictionary.\")\n raise ValueError(\"Task parameters must be a dictionary.\")\n\n logger.reason(\"Creating task node and scheduling execution\")\n task = Task(plugin_id=plugin_id, params=params, user_id=user_id)\n self.tasks[task.id] = task\n self.persistence_service.persist_task(task)\n logger.info(f\"Task {task.id} created and scheduled for execution\")\n self.loop.create_task(\n self._run_task(task.id)\n ) # Schedule task for execution\n logger.reflect(\n \"Task creation persisted and execution scheduled\",\n extra={\"task_id\": task.id, \"plugin_id\": plugin_id},\n )\n return task\n\n # [/DEF:create_task:Function]\n\n # [DEF:_run_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Internal method to execute a task with TaskContext support.\n # @PRE: Task exists in registry.\n # @POST: Task is executed, status updated to SUCCESS or FAILED.\n # @PARAM: task_id (str) - The ID of the task to run.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def _run_task(self, task_id: str):\n with belief_scope(\"TaskManager._run_task\", f\"task_id={task_id}\"):\n task = self.tasks[task_id]\n plugin = self.plugin_loader.get_plugin(task.plugin_id)\n\n logger.reason(\n \"Transitioning task to running state\",\n extra={\"task_id\": task_id, \"plugin_id\": task.plugin_id},\n )\n logger.info(\n f\"Starting execution of task {task_id} for plugin '{plugin.name}'\"\n )\n task.status = TaskStatus.RUNNING\n task.started_at = datetime.utcnow()\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task started for plugin '{plugin.name}'\",\n source=\"system\",\n )\n\n try:\n # Prepare params and check if plugin supports new TaskContext\n params = {**task.params, \"_task_id\": task_id}\n\n # Check if plugin's execute method accepts 'context' parameter\n sig = inspect.signature(plugin.execute)\n accepts_context = \"context\" in sig.parameters\n\n if accepts_context:\n # Create TaskContext for new-style plugins\n context = TaskContext(\n task_id=task_id,\n add_log_fn=self._add_log,\n params=params,\n default_source=\"plugin\",\n background_tasks=None,\n )\n\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params, context=context)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor,\n lambda: plugin.execute(params, context=context),\n )\n else:\n # Backward compatibility: old-style plugins without context\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor, plugin.execute, params\n )\n\n logger.info(f\"Task {task_id} completed successfully\")\n task.status = TaskStatus.SUCCESS\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task completed successfully for plugin '{plugin.name}'\",\n source=\"system\",\n )\n except Exception as e:\n logger.error(f\"Task {task_id} failed: {e}\")\n task.status = TaskStatus.FAILED\n self._add_log(\n task_id,\n \"ERROR\",\n f\"Task failed: {e}\",\n source=\"system\",\n metadata={\"error_type\": type(e).__name__},\n )\n finally:\n task.finished_at = datetime.utcnow()\n # Flush any remaining buffered logs before persisting task\n self._flush_task_logs(task_id)\n self.persistence_service.persist_task(task)\n logger.info(\n f\"Task {task_id} execution finished with status: {task.status}\"\n )\n logger.reflect(\n \"Task lifecycle reached persisted terminal state\",\n extra={\"task_id\": task_id, \"status\": str(task.status)},\n )\n\n # [/DEF:_run_task:Function]\n\n # [DEF:resolve_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resumes a task that is awaiting mapping.\n # @PRE: Task exists and is in AWAITING_MAPPING state.\n # @POST: Task status updated to RUNNING, params updated, execution resumed.\n # @PARAM: task_id (str) - The ID of the task.\n # @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.\n # @THROWS: ValueError if task not found or not awaiting mapping.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):\n with belief_scope(\"TaskManager.resolve_task\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task or task.status != TaskStatus.AWAITING_MAPPING:\n raise ValueError(\"Task is not awaiting mapping.\")\n\n # Update task params with resolution\n task.params.update(resolution_params)\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(task_id, \"INFO\", \"Task resumed after mapping resolution.\")\n\n # Signal the future to continue\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resolve_task:Function]\n\n # [DEF:wait_for_resolution:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for a resolution signal.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_resolution(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_resolution\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n task.status = TaskStatus.AWAITING_MAPPING\n self.persistence_service.persist_task(task)\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_resolution:Function]\n\n # [DEF:wait_for_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for user input.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set via resume_task_with_password.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_input(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Status is already set to AWAITING_INPUT by await_input()\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_input:Function]\n\n # [DEF:get_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves a task by its ID.\n # @PRE: task_id is a string.\n # @POST: Returns Task object or None.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: Optional[Task] - The task or None.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_task(self, task_id: str) -> Optional[Task]:\n with belief_scope(\"TaskManager.get_task\", f\"task_id={task_id}\"):\n return self.tasks.get(task_id)\n\n # [/DEF:get_task:Function]\n\n # [DEF:get_all_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves all registered tasks.\n # @PRE: None.\n # @POST: Returns list of all Task objects.\n # @RETURN: List[Task] - All tasks.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_all_tasks(self) -> List[Task]:\n with belief_scope(\"TaskManager.get_all_tasks\"):\n return list(self.tasks.values())\n\n # [/DEF:get_all_tasks:Function]\n\n # [DEF:get_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves tasks with pagination and optional status filter.\n # @PRE: limit and offset are non-negative integers.\n # @POST: Returns a list of tasks sorted by start_time descending.\n # @PARAM: limit (int) - Maximum number of tasks to return.\n # @PARAM: offset (int) - Number of tasks to skip.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: List[Task] - List of tasks matching criteria.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_tasks(\n self,\n limit: int = 10,\n offset: int = 0,\n status: Optional[TaskStatus] = None,\n plugin_ids: Optional[List[str]] = None,\n completed_only: bool = False,\n ) -> List[Task]:\n with belief_scope(\"TaskManager.get_tasks\"):\n tasks = list(self.tasks.values())\n if status:\n tasks = [t for t in tasks if t.status == status]\n if plugin_ids:\n plugin_id_set = set(plugin_ids)\n tasks = [t for t in tasks if t.plugin_id in plugin_id_set]\n if completed_only:\n tasks = [\n t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]\n ]\n\n # Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.\n def sort_key(task: Task) -> float:\n started_at = task.started_at\n if started_at is None:\n return float(\"-inf\")\n if not isinstance(started_at, datetime):\n return float(\"-inf\")\n if started_at.tzinfo is None:\n return started_at.replace(tzinfo=timezone.utc).timestamp()\n return started_at.timestamp()\n\n tasks.sort(key=sort_key, reverse=True)\n return tasks[offset : offset + limit]\n\n # [/DEF:get_tasks:Function]\n\n # [DEF:get_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).\n # @PRE: task_id is a string.\n # @POST: Returns list of LogEntry or TaskLog objects.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.\n # @RETURN: List[LogEntry] - List of log entries.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_logs(\n self, task_id: str, log_filter: Optional[LogFilter] = None\n ) -> List[LogEntry]:\n with belief_scope(\"TaskManager.get_task_logs\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n\n # For completed tasks, fetch from persistence\n if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:\n if log_filter is None:\n log_filter = LogFilter()\n task_logs = self.log_persistence_service.get_logs(task_id, log_filter)\n # Convert TaskLog to LogEntry for backward compatibility\n return [\n LogEntry(\n timestamp=log.timestamp,\n level=log.level,\n message=log.message,\n source=log.source,\n metadata=log.metadata,\n )\n for log in task_logs\n ]\n\n # For running/pending tasks, return from memory\n return task.logs if task else []\n\n # [/DEF:get_task_logs:Function]\n\n # [DEF:get_task_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\"TaskManager.get_task_log_stats\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_log_stats(task_id)\n\n # [/DEF:get_task_log_stats:Function]\n\n # [DEF:get_task_log_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_sources(self, task_id: str) -> List[str]:\n with belief_scope(\"TaskManager.get_task_log_sources\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_sources(task_id)\n\n # [/DEF:get_task_log_sources:Function]\n\n # [DEF:_add_log:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.\n # @PRE: Task exists.\n # @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: level (str) - Log level.\n # @PARAM: message (str) - Log message.\n # @PARAM: source (str) - Source component (default: \"system\").\n # @PARAM: metadata (Optional[Dict]) - Additional structured data.\n # @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).\n # @RELATION: [CALLS] ->[should_log_task_level]\n # @RELATION: [DISPATCHES] ->[EventBus]\n def _add_log(\n self,\n task_id: str,\n level: str,\n message: str,\n source: str = \"system\",\n metadata: Optional[Dict[str, Any]] = None,\n context: Optional[Dict[str, Any]] = None,\n ):\n with belief_scope(\"TaskManager._add_log\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Filter logs based on task_log_level configuration\n if not should_log_task_level(level):\n return\n\n # Create log entry with new fields\n log_entry = LogEntry(\n level=level,\n message=message,\n source=source,\n metadata=metadata,\n context=context, # Keep for backward compatibility\n )\n\n # Add to in-memory logs (for backward compatibility with legacy JSON field)\n task.logs.append(log_entry)\n\n # Add to buffer for batch persistence\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].append(log_entry)\n\n # Notify subscribers (for real-time WebSocket updates)\n if task_id in self.subscribers:\n for queue in self.subscribers[task_id]:\n self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)\n\n # [/DEF:_add_log:Function]\n\n # [DEF:subscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Subscribes to real-time logs for a task.\n # @PRE: task_id is a string.\n # @POST: Returns an asyncio.Queue for log entries.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: asyncio.Queue - Queue for log entries.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def subscribe_logs(self, task_id: str) -> asyncio.Queue:\n with belief_scope(\"TaskManager.subscribe_logs\", f\"task_id={task_id}\"):\n queue = asyncio.Queue()\n if task_id not in self.subscribers:\n self.subscribers[task_id] = []\n self.subscribers[task_id].append(queue)\n return queue\n\n # [/DEF:subscribe_logs:Function]\n\n # [DEF:unsubscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Unsubscribes from real-time logs for a task.\n # @PRE: task_id is a string, queue is asyncio.Queue.\n # @POST: Queue removed from subscribers.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: queue (asyncio.Queue) - Queue to remove.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):\n with belief_scope(\"TaskManager.unsubscribe_logs\", f\"task_id={task_id}\"):\n if task_id in self.subscribers:\n if queue in self.subscribers[task_id]:\n self.subscribers[task_id].remove(queue)\n if not self.subscribers[task_id]:\n del self.subscribers[task_id]\n\n # [/DEF:unsubscribe_logs:Function]\n\n # [DEF:load_persisted_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Load persisted tasks using persistence service.\n # @PRE: None.\n # @POST: Persisted tasks loaded into self.tasks.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def load_persisted_tasks(self) -> None:\n with belief_scope(\"TaskManager.load_persisted_tasks\"):\n loaded_tasks = self.persistence_service.load_tasks(limit=100)\n for task in loaded_tasks:\n if task.id not in self.tasks:\n self.tasks[task.id] = task\n\n # [/DEF:load_persisted_tasks:Function]\n\n # [DEF:await_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Transition a task to AWAITING_INPUT state with input request.\n # @PRE: Task exists and is in RUNNING state.\n # @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: input_request (Dict) - Details about required input.\n # @THROWS: ValueError if task not found or not RUNNING.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:\n with belief_scope(\"TaskManager.await_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.RUNNING:\n raise ValueError(\n f\"Task {task_id} is not RUNNING (current: {task.status})\"\n )\n\n task.status = TaskStatus.AWAITING_INPUT\n task.input_required = True\n task.input_request = input_request\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task paused for user input\",\n metadata={\"input_request\": input_request},\n )\n\n # [/DEF:await_input:Function]\n\n # [DEF:resume_task_with_password:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resume a task that is awaiting input with provided passwords.\n # @PRE: Task exists and is in AWAITING_INPUT state.\n # @POST: Task status changed to RUNNING, passwords injected, task resumed.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.\n # @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def resume_task_with_password(\n self, task_id: str, passwords: Dict[str, str]\n ) -> None:\n with belief_scope(\n \"TaskManager.resume_task_with_password\", f\"task_id={task_id}\"\n ):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.AWAITING_INPUT:\n raise ValueError(\n f\"Task {task_id} is not AWAITING_INPUT (current: {task.status})\"\n )\n\n if not isinstance(passwords, dict) or not passwords:\n raise ValueError(\"Passwords must be a non-empty dictionary\")\n\n task.params[\"passwords\"] = passwords\n task.input_required = False\n task.input_request = None\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task resumed with passwords\",\n metadata={\"databases\": list(passwords.keys())},\n )\n\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resume_task_with_password:Function]\n\n # [DEF:clear_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Clears tasks based on status filter (also deletes associated logs).\n # @PRE: status is Optional[TaskStatus].\n # @POST: Tasks matching filter (or all non-active) cleared from registry and database.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: int - Number of tasks cleared.\n # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:\n with belief_scope(\"TaskManager.clear_tasks\"):\n tasks_to_remove = []\n for task_id, task in list(self.tasks.items()):\n # If status is provided, match it.\n # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)\n # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.\n # RUNNING is active execution.\n\n should_remove = False\n if status:\n if task.status == status:\n should_remove = True\n else:\n # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)\n if task.status not in [\n TaskStatus.RUNNING,\n TaskStatus.AWAITING_INPUT,\n TaskStatus.AWAITING_MAPPING,\n ]:\n should_remove = True\n\n if should_remove:\n tasks_to_remove.append(task_id)\n\n for tid in tasks_to_remove:\n # Cancel future if exists (e.g. for AWAITING_INPUT/MAPPING)\n if tid in self.task_futures:\n self.task_futures[tid].cancel()\n del self.task_futures[tid]\n\n del self.tasks[tid]\n\n # Remove from persistence (task_records and task_logs via CASCADE)\n self.persistence_service.delete_tasks(tasks_to_remove)\n\n # Also explicitly delete logs (in case CASCADE is not set up)\n if tasks_to_remove:\n self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)\n\n logger.info(f\"Cleared {len(tasks_to_remove)} tasks.\")\n return len(tasks_to_remove)\n\n # [/DEF:clear_tasks:Function]\n\n\n# [/DEF:TaskManager:Class]\n" + "body": "# [DEF:TaskManager:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: task, manager, lifecycle, execution, state\n# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n# @LAYER: Core\n# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n# @RELATION: [DEPENDS_ON] ->[PluginLoader]\n# @RELATION: [DEPENDS_ON] ->[TaskContext]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.\n# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.\n# @INVARIANT: Task IDs are unique within the registry.\n# @INVARIANT: Each task has exactly one status at any time.\n# @INVARIANT: Log entries are never deleted after being added to a task.\n# @SIDE_EFFECT: Spawns worker threads, flushes logs to database, and mutates task states.\n# @DATA_CONTRACT: Input[plugin_id, params] -> Output[Task]\nclass TaskManager:\n \"\"\"\n Manages the lifecycle of tasks, including their creation, execution, and state tracking.\n \"\"\"\n\n # Log flush interval in seconds\n LOG_FLUSH_INTERVAL = 2.0\n\n # [DEF:TaskGraph:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.\n # @RELATION: [DEPENDS_ON] ->[Task]\n # @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]\n # @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.\n # @POST: Each live task id resolves to at most one active Task node and optional pause future.\n # @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.\n # @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]\n # @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.\n # [/DEF:TaskGraph:Block]\n\n # [DEF:EventBus:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.\n # @RELATION: [DEPENDS_ON] ->[LogEntry]\n # @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]\n # @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.\n # @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.\n # @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.\n # @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]\n # @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.\n # [/DEF:EventBus:Block]\n\n # [DEF:JobLifecycle:Block]\n # @COMPLEXITY: 5\n # @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[PluginLoader]\n # @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.\n # @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.\n # @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.\n # @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]\n # @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.\n # [/DEF:JobLifecycle:Block]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Initialize the TaskManager with dependencies.\n # @PRE: plugin_loader is initialized.\n # @POST: TaskManager is ready to accept tasks.\n # @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n # @PARAM: plugin_loader - The plugin loader instance.\n def __init__(self, plugin_loader):\n with belief_scope(\"TaskManager.__init__\"):\n log.reason(\"Initializing task manager runtime services\")\n self.plugin_loader = plugin_loader\n self.tasks: Dict[str, Task] = {}\n self.subscribers: Dict[str, List[asyncio.Queue]] = {}\n self.executor = ThreadPoolExecutor(\n max_workers=5\n ) # For CPU-bound plugin execution\n self.persistence_service = TaskPersistenceService()\n self.log_persistence_service = TaskLogPersistenceService()\n\n # Log buffer: task_id -> List[LogEntry]\n self._log_buffer: Dict[str, List[LogEntry]] = {}\n self._log_buffer_lock = threading.Lock()\n\n # Flusher thread for batch writing logs\n self._flusher_stop_event = threading.Event()\n self._flusher_thread = threading.Thread(\n target=self._flusher_loop, daemon=True\n )\n self._flusher_thread.start()\n\n try:\n self.loop = asyncio.get_running_loop()\n except RuntimeError:\n self.loop = asyncio.get_event_loop()\n self.task_futures: Dict[str, asyncio.Future] = {}\n\n # Load persisted tasks on startup\n self.load_persisted_tasks()\n log.reflect(\n \"Task manager runtime initialized\",\n payload={\"task_count\": len(self.tasks)},\n )\n\n # [/DEF:__init__:Function]\n\n # [DEF:_flusher_loop:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Background thread that periodically flushes log buffer to database.\n # @PRE: TaskManager is initialized.\n # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.\n # @RELATION: [CALLS] ->[TaskManager._flush_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flusher_loop(self):\n \"\"\"Background thread that flushes log buffer to database.\"\"\"\n while not self._flusher_stop_event.is_set():\n self._flush_logs()\n self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)\n\n # [/DEF:_flusher_loop:Function]\n\n # [DEF:_flush_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush all buffered logs to the database.\n # @PRE: None.\n # @POST: All buffered logs are written to task_logs table.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_logs(self):\n \"\"\"Flush all buffered logs to the database.\"\"\"\n with self._log_buffer_lock:\n task_ids = list(self._log_buffer.keys())\n\n for task_id in task_ids:\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n log.reflect(\"Flushed logs to database\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n log.explore(\"Failed to flush logs to database\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n # Re-add logs to buffer on failure\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].extend(logs)\n\n # [/DEF:_flush_logs:Function]\n\n # [DEF:_flush_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush logs for a specific task immediately.\n # @PRE: task_id exists.\n # @POST: Task's buffered logs are written to database.\n # @PARAM: task_id (str) - The task ID.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_task_logs(self, task_id: str):\n \"\"\"Flush logs for a specific task immediately.\"\"\"\n with belief_scope(\"_flush_task_logs\"):\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n log.reflect(\"Flushed task logs immediately\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n log.explore(\"Failed to flush task logs immediately\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n\n # [/DEF:_flush_task_logs:Function]\n\n # [DEF:create_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Creates and queues a new task for execution.\n # @PRE: Plugin with plugin_id exists. Params are valid.\n # @POST: Task is created, added to registry, and scheduled for execution.\n # @PARAM: plugin_id (str) - The ID of the plugin to run.\n # @PARAM: params (Dict[str, Any]) - Parameters for the plugin.\n # @PARAM: user_id (Optional[str]) - ID of the user requesting the task.\n # @RETURN: Task - The created task instance.\n # @THROWS: ValueError if plugin not found or params invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n async def create_task(\n self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None\n ) -> Task:\n with belief_scope(\"TaskManager.create_task\", f\"plugin_id={plugin_id}\"):\n if not self.plugin_loader.has_plugin(plugin_id):\n log.explore(\"Plugin not found\", error=f\"Plugin with ID '{plugin_id}' not found.\")\n raise ValueError(f\"Plugin with ID '{plugin_id}' not found.\")\n\n self.plugin_loader.get_plugin(plugin_id)\n\n if not isinstance(params, dict):\n log.explore(\"Invalid task params type\", error=\"Task parameters must be a dictionary.\")\n raise ValueError(\"Task parameters must be a dictionary.\")\n\n log.reason(\"Creating task node and scheduling execution\", payload={\"plugin_id\": plugin_id})\n task = Task(plugin_id=plugin_id, params=params, user_id=user_id)\n self.tasks[task.id] = task\n self.persistence_service.persist_task(task)\n trace_id = get_trace_id()\n self.loop.create_task(\n self._run_task(task.id, trace_id=trace_id)\n ) # Schedule task for execution\n log.reflect(\n \"Task creation persisted and execution scheduled\",\n payload={\"task_id\": task.id, \"plugin_id\": plugin_id},\n )\n return task\n\n # [/DEF:create_task:Function]\n\n # [DEF:_run_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Internal method to execute a task with TaskContext support.\n # @PRE: Task exists in registry.\n # @POST: Task is executed, status updated to SUCCESS or FAILED.\n # @PARAM: task_id (str) - The ID of the task to run.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def _run_task(self, task_id: str, trace_id: str = None):\n if trace_id:\n set_trace_id(trace_id)\n with belief_scope(\"TaskManager._run_task\", f\"task_id={task_id}\"):\n task = self.tasks[task_id]\n plugin = self.plugin_loader.get_plugin(task.plugin_id)\n\n log.reason(\n \"Transitioning task to running state\",\n payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id},\n )\n task.status = TaskStatus.RUNNING\n task.started_at = datetime.utcnow()\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task started for plugin '{plugin.name}'\",\n source=\"system\",\n )\n\n try:\n # Prepare params and check if plugin supports new TaskContext\n params = {**task.params, \"_task_id\": task_id}\n\n # Check if plugin's execute method accepts 'context' parameter\n sig = inspect.signature(plugin.execute)\n accepts_context = \"context\" in sig.parameters\n\n if accepts_context:\n # Create TaskContext for new-style plugins\n context = TaskContext(\n task_id=task_id,\n add_log_fn=self._add_log,\n params=params,\n default_source=\"plugin\",\n background_tasks=None,\n )\n\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params, context=context)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor,\n lambda: plugin.execute(params, context=context),\n )\n else:\n # Backward compatibility: old-style plugins without context\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor, plugin.execute, params\n )\n\n task.status = TaskStatus.SUCCESS\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task completed successfully for plugin '{plugin.name}'\",\n source=\"system\",\n )\n log.reflect(\"Task completed successfully\", payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id})\n except Exception as e:\n task.status = TaskStatus.FAILED\n self._add_log(\n task_id,\n \"ERROR\",\n f\"Task failed: {e}\",\n source=\"system\",\n metadata={\"error_type\": type(e).__name__},\n )\n log.explore(\"Task failed during execution\", error=str(e), payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id})\n finally:\n task.finished_at = datetime.utcnow()\n # Flush any remaining buffered logs before persisting task\n self._flush_task_logs(task_id)\n self.persistence_service.persist_task(task)\n log.reflect(\n \"Task execution finished\",\n payload={\"task_id\": task_id, \"status\": str(task.status)},\n )\n\n # [/DEF:_run_task:Function]\n\n # [DEF:resolve_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resumes a task that is awaiting mapping.\n # @PRE: Task exists and is in AWAITING_MAPPING state.\n # @POST: Task status updated to RUNNING, params updated, execution resumed.\n # @PARAM: task_id (str) - The ID of the task.\n # @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.\n # @THROWS: ValueError if task not found or not awaiting mapping.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):\n with belief_scope(\"TaskManager.resolve_task\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task or task.status != TaskStatus.AWAITING_MAPPING:\n raise ValueError(\"Task is not awaiting mapping.\")\n\n # Update task params with resolution\n task.params.update(resolution_params)\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(task_id, \"INFO\", \"Task resumed after mapping resolution.\")\n\n # Signal the future to continue\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resolve_task:Function]\n\n # [DEF:wait_for_resolution:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for a resolution signal.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_resolution(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_resolution\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n task.status = TaskStatus.AWAITING_MAPPING\n self.persistence_service.persist_task(task)\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_resolution:Function]\n\n # [DEF:wait_for_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Pauses execution and waits for user input.\n # @PRE: Task exists.\n # @POST: Execution pauses until future is set via resume_task_with_password.\n # @PARAM: task_id (str) - The ID of the task.\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n async def wait_for_input(self, task_id: str):\n with belief_scope(\"TaskManager.wait_for_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Status is already set to AWAITING_INPUT by await_input()\n self.task_futures[task_id] = self.loop.create_future()\n\n try:\n await self.task_futures[task_id]\n finally:\n if task_id in self.task_futures:\n del self.task_futures[task_id]\n\n # [/DEF:wait_for_input:Function]\n\n # [DEF:get_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves a task by its ID.\n # @PRE: task_id is a string.\n # @POST: Returns Task object or None.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: Optional[Task] - The task or None.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_task(self, task_id: str) -> Optional[Task]:\n with belief_scope(\"TaskManager.get_task\", f\"task_id={task_id}\"):\n return self.tasks.get(task_id)\n\n # [/DEF:get_task:Function]\n\n # [DEF:get_all_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves all registered tasks.\n # @PRE: None.\n # @POST: Returns list of all Task objects.\n # @RETURN: List[Task] - All tasks.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_all_tasks(self) -> List[Task]:\n with belief_scope(\"TaskManager.get_all_tasks\"):\n return list(self.tasks.values())\n\n # [/DEF:get_all_tasks:Function]\n\n # [DEF:get_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves tasks with pagination and optional status filter.\n # @PRE: limit and offset are non-negative integers.\n # @POST: Returns a list of tasks sorted by start_time descending.\n # @PARAM: limit (int) - Maximum number of tasks to return.\n # @PARAM: offset (int) - Number of tasks to skip.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: List[Task] - List of tasks matching criteria.\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def get_tasks(\n self,\n limit: int = 10,\n offset: int = 0,\n status: Optional[TaskStatus] = None,\n plugin_ids: Optional[List[str]] = None,\n completed_only: bool = False,\n ) -> List[Task]:\n with belief_scope(\"TaskManager.get_tasks\"):\n tasks = list(self.tasks.values())\n if status:\n tasks = [t for t in tasks if t.status == status]\n if plugin_ids:\n plugin_id_set = set(plugin_ids)\n tasks = [t for t in tasks if t.plugin_id in plugin_id_set]\n if completed_only:\n tasks = [\n t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]\n ]\n\n # Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.\n def sort_key(task: Task) -> float:\n started_at = task.started_at\n if started_at is None:\n return float(\"-inf\")\n if not isinstance(started_at, datetime):\n return float(\"-inf\")\n if started_at.tzinfo is None:\n return started_at.replace(tzinfo=timezone.utc).timestamp()\n return started_at.timestamp()\n\n tasks.sort(key=sort_key, reverse=True)\n return tasks[offset : offset + limit]\n\n # [/DEF:get_tasks:Function]\n\n # [DEF:get_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).\n # @PRE: task_id is a string.\n # @POST: Returns list of LogEntry or TaskLog objects.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.\n # @RETURN: List[LogEntry] - List of log entries.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_logs(\n self, task_id: str, log_filter: Optional[LogFilter] = None\n ) -> List[LogEntry]:\n with belief_scope(\"TaskManager.get_task_logs\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n\n # For completed tasks, fetch from persistence\n if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:\n if log_filter is None:\n log_filter = LogFilter()\n task_logs = self.log_persistence_service.get_logs(task_id, log_filter)\n # Convert TaskLog to LogEntry for backward compatibility\n return [\n LogEntry(\n timestamp=log.timestamp,\n level=log.level,\n message=log.message,\n source=log.source,\n metadata=log.metadata,\n )\n for log in task_logs\n ]\n\n # For running/pending tasks, return from memory\n return task.logs if task else []\n\n # [/DEF:get_task_logs:Function]\n\n # [DEF:get_task_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\"TaskManager.get_task_log_stats\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_log_stats(task_id)\n\n # [/DEF:get_task_log_stats:Function]\n\n # [DEF:get_task_log_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def get_task_log_sources(self, task_id: str) -> List[str]:\n with belief_scope(\"TaskManager.get_task_log_sources\", f\"task_id={task_id}\"):\n return self.log_persistence_service.get_sources(task_id)\n\n # [/DEF:get_task_log_sources:Function]\n\n # [DEF:_add_log:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.\n # @PRE: Task exists.\n # @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: level (str) - Log level.\n # @PARAM: message (str) - Log message.\n # @PARAM: source (str) - Source component (default: \"system\").\n # @PARAM: metadata (Optional[Dict]) - Additional structured data.\n # @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).\n # @RELATION: [CALLS] ->[should_log_task_level]\n # @RELATION: [DISPATCHES] ->[EventBus]\n def _add_log(\n self,\n task_id: str,\n level: str,\n message: str,\n source: str = \"system\",\n metadata: Optional[Dict[str, Any]] = None,\n context: Optional[Dict[str, Any]] = None,\n ):\n with belief_scope(\"TaskManager._add_log\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n return\n\n # Filter logs based on task_log_level configuration\n if not should_log_task_level(level):\n return\n\n # Create log entry with new fields\n log_entry = LogEntry(\n level=level,\n message=message,\n source=source,\n metadata=metadata,\n context=context, # Keep for backward compatibility\n )\n\n # Add to in-memory logs (for backward compatibility with legacy JSON field)\n task.logs.append(log_entry)\n\n # Add to buffer for batch persistence\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].append(log_entry)\n\n # Notify subscribers (for real-time WebSocket updates)\n if task_id in self.subscribers:\n for queue in self.subscribers[task_id]:\n self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)\n\n # [/DEF:_add_log:Function]\n\n # [DEF:subscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Subscribes to real-time logs for a task.\n # @PRE: task_id is a string.\n # @POST: Returns an asyncio.Queue for log entries.\n # @PARAM: task_id (str) - ID of the task.\n # @RETURN: asyncio.Queue - Queue for log entries.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def subscribe_logs(self, task_id: str) -> asyncio.Queue:\n with belief_scope(\"TaskManager.subscribe_logs\", f\"task_id={task_id}\"):\n queue = asyncio.Queue()\n if task_id not in self.subscribers:\n self.subscribers[task_id] = []\n self.subscribers[task_id].append(queue)\n return queue\n\n # [/DEF:subscribe_logs:Function]\n\n # [DEF:unsubscribe_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Unsubscribes from real-time logs for a task.\n # @PRE: task_id is a string, queue is asyncio.Queue.\n # @POST: Queue removed from subscribers.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: queue (asyncio.Queue) - Queue to remove.\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):\n with belief_scope(\"TaskManager.unsubscribe_logs\", f\"task_id={task_id}\"):\n if task_id in self.subscribers:\n if queue in self.subscribers[task_id]:\n self.subscribers[task_id].remove(queue)\n if not self.subscribers[task_id]:\n del self.subscribers[task_id]\n\n # [/DEF:unsubscribe_logs:Function]\n\n # [DEF:load_persisted_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Load persisted tasks using persistence service.\n # @PRE: None.\n # @POST: Persisted tasks loaded into self.tasks.\n # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n def load_persisted_tasks(self) -> None:\n with belief_scope(\"TaskManager.load_persisted_tasks\"):\n loaded_tasks = self.persistence_service.load_tasks(limit=100)\n for task in loaded_tasks:\n if task.id not in self.tasks:\n self.tasks[task.id] = task\n\n # [/DEF:load_persisted_tasks:Function]\n\n # [DEF:await_input:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Transition a task to AWAITING_INPUT state with input request.\n # @PRE: Task exists and is in RUNNING state.\n # @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: input_request (Dict) - Details about required input.\n # @THROWS: ValueError if task not found or not RUNNING.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:\n with belief_scope(\"TaskManager.await_input\", f\"task_id={task_id}\"):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.RUNNING:\n raise ValueError(\n f\"Task {task_id} is not RUNNING (current: {task.status})\"\n )\n\n task.status = TaskStatus.AWAITING_INPUT\n task.input_required = True\n task.input_request = input_request\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task paused for user input\",\n metadata={\"input_request\": input_request},\n )\n\n # [/DEF:await_input:Function]\n\n # [DEF:resume_task_with_password:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resume a task that is awaiting input with provided passwords.\n # @PRE: Task exists and is in AWAITING_INPUT state.\n # @POST: Task status changed to RUNNING, passwords injected, task resumed.\n # @PARAM: task_id (str) - ID of the task.\n # @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.\n # @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n def resume_task_with_password(\n self, task_id: str, passwords: Dict[str, str]\n ) -> None:\n with belief_scope(\n \"TaskManager.resume_task_with_password\", f\"task_id={task_id}\"\n ):\n task = self.tasks.get(task_id)\n if not task:\n raise ValueError(f\"Task {task_id} not found\")\n if task.status != TaskStatus.AWAITING_INPUT:\n raise ValueError(\n f\"Task {task_id} is not AWAITING_INPUT (current: {task.status})\"\n )\n\n if not isinstance(passwords, dict) or not passwords:\n raise ValueError(\"Passwords must be a non-empty dictionary\")\n\n task.params[\"passwords\"] = passwords\n task.input_required = False\n task.input_request = None\n task.status = TaskStatus.RUNNING\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n \"Task resumed with passwords\",\n metadata={\"databases\": list(passwords.keys())},\n )\n\n if task_id in self.task_futures:\n self.task_futures[task_id].set_result(True)\n\n # [/DEF:resume_task_with_password:Function]\n\n # [DEF:clear_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Clears tasks based on status filter (also deletes associated logs).\n # @PRE: status is Optional[TaskStatus].\n # @POST: Tasks matching filter (or all non-active) cleared from registry and database.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: int - Number of tasks cleared.\n # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:\n with belief_scope(\"TaskManager.clear_tasks\"):\n tasks_to_remove = []\n for task_id, task in list(self.tasks.items()):\n # If status is provided, match it.\n # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)\n # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.\n # RUNNING is active execution.\n\n should_remove = False\n if status:\n if task.status == status:\n should_remove = True\n else:\n # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)\n if task.status not in [\n TaskStatus.RUNNING,\n TaskStatus.AWAITING_INPUT,\n TaskStatus.AWAITING_MAPPING,\n ]:\n should_remove = True\n\n if should_remove:\n tasks_to_remove.append(task_id)\n\n for tid in tasks_to_remove:\n # Cancel future if exists (e.g. for AWAITING_INPUT/MAPPING)\n if tid in self.task_futures:\n self.task_futures[tid].cancel()\n del self.task_futures[tid]\n\n del self.tasks[tid]\n\n # Remove from persistence (task_records and task_logs via CASCADE)\n self.persistence_service.delete_tasks(tasks_to_remove)\n\n # Also explicitly delete logs (in case CASCADE is not set up)\n if tasks_to_remove:\n self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)\n\n log.reflect(\"Tasks cleared from registry and database\", payload={\"count\": len(tasks_to_remove)})\n return len(tasks_to_remove)\n\n # [/DEF:clear_tasks:Function]\n\n\n# [/DEF:TaskManager:Class]\n" }, { "contract_id": "TaskGraph", "contract_type": "Block", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 72, - "end_line": 82, + "start_line": 75, + "end_line": 85, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -42781,8 +43537,8 @@ "contract_id": "EventBus", "contract_type": "Block", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 84, - "end_line": 94, + "start_line": 87, + "end_line": 97, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -42966,8 +43722,8 @@ "contract_id": "JobLifecycle", "contract_type": "Block", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 96, - "end_line": 108, + "start_line": 99, + "end_line": 111, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -43197,8 +43953,8 @@ "contract_id": "_flusher_loop", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 158, - "end_line": 171, + "start_line": 161, + "end_line": 174, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43282,8 +44038,8 @@ "contract_id": "_flush_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 173, - "end_line": 201, + "start_line": 176, + "end_line": 205, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43361,14 +44117,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_flush_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush all buffered logs to the database.\n # @PRE: None.\n # @POST: All buffered logs are written to task_logs table.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_logs(self):\n \"\"\"Flush all buffered logs to the database.\"\"\"\n with self._log_buffer_lock:\n task_ids = list(self._log_buffer.keys())\n\n for task_id in task_ids:\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n logger.debug(f\"Flushed {len(logs)} logs for task {task_id}\")\n except Exception as e:\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n # Re-add logs to buffer on failure\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].extend(logs)\n\n # [/DEF:_flush_logs:Function]\n" + "body": " # [DEF:_flush_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush all buffered logs to the database.\n # @PRE: None.\n # @POST: All buffered logs are written to task_logs table.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_logs(self):\n \"\"\"Flush all buffered logs to the database.\"\"\"\n with self._log_buffer_lock:\n task_ids = list(self._log_buffer.keys())\n\n for task_id in task_ids:\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n log.reflect(\"Flushed logs to database\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n log.explore(\"Failed to flush logs to database\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n # Re-add logs to buffer on failure\n with self._log_buffer_lock:\n if task_id not in self._log_buffer:\n self._log_buffer[task_id] = []\n self._log_buffer[task_id].extend(logs)\n\n # [/DEF:_flush_logs:Function]\n" }, { "contract_id": "_flush_task_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 203, - "end_line": 223, + "start_line": 207, + "end_line": 229, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43453,14 +44209,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_flush_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush logs for a specific task immediately.\n # @PRE: task_id exists.\n # @POST: Task's buffered logs are written to database.\n # @PARAM: task_id (str) - The task ID.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_task_logs(self, task_id: str):\n \"\"\"Flush logs for a specific task immediately.\"\"\"\n with belief_scope(\"_flush_task_logs\"):\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n except Exception as e:\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n\n # [/DEF:_flush_task_logs:Function]\n" + "body": " # [DEF:_flush_task_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Flush logs for a specific task immediately.\n # @PRE: task_id exists.\n # @POST: Task's buffered logs are written to database.\n # @PARAM: task_id (str) - The task ID.\n # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def _flush_task_logs(self, task_id: str):\n \"\"\"Flush logs for a specific task immediately.\"\"\"\n with belief_scope(\"_flush_task_logs\"):\n with self._log_buffer_lock:\n logs = self._log_buffer.pop(task_id, [])\n\n if logs:\n try:\n self.log_persistence_service.add_logs(task_id, logs)\n log.reflect(\"Flushed task logs immediately\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n log.explore(\"Failed to flush task logs immediately\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to flush logs for task {task_id}: {e}\")\n\n # [/DEF:_flush_task_logs:Function]\n" }, { "contract_id": "create_task", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 225, - "end_line": 266, + "start_line": 231, + "end_line": 272, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43582,14 +44338,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:create_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Creates and queues a new task for execution.\n # @PRE: Plugin with plugin_id exists. Params are valid.\n # @POST: Task is created, added to registry, and scheduled for execution.\n # @PARAM: plugin_id (str) - The ID of the plugin to run.\n # @PARAM: params (Dict[str, Any]) - Parameters for the plugin.\n # @PARAM: user_id (Optional[str]) - ID of the user requesting the task.\n # @RETURN: Task - The created task instance.\n # @THROWS: ValueError if plugin not found or params invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n async def create_task(\n self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None\n ) -> Task:\n with belief_scope(\"TaskManager.create_task\", f\"plugin_id={plugin_id}\"):\n if not self.plugin_loader.has_plugin(plugin_id):\n logger.error(f\"Plugin with ID '{plugin_id}' not found.\")\n raise ValueError(f\"Plugin with ID '{plugin_id}' not found.\")\n\n self.plugin_loader.get_plugin(plugin_id)\n\n if not isinstance(params, dict):\n logger.error(\"Task parameters must be a dictionary.\")\n raise ValueError(\"Task parameters must be a dictionary.\")\n\n logger.reason(\"Creating task node and scheduling execution\")\n task = Task(plugin_id=plugin_id, params=params, user_id=user_id)\n self.tasks[task.id] = task\n self.persistence_service.persist_task(task)\n logger.info(f\"Task {task.id} created and scheduled for execution\")\n self.loop.create_task(\n self._run_task(task.id)\n ) # Schedule task for execution\n logger.reflect(\n \"Task creation persisted and execution scheduled\",\n extra={\"task_id\": task.id, \"plugin_id\": plugin_id},\n )\n return task\n\n # [/DEF:create_task:Function]\n" + "body": " # [DEF:create_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Creates and queues a new task for execution.\n # @PRE: Plugin with plugin_id exists. Params are valid.\n # @POST: Task is created, added to registry, and scheduled for execution.\n # @PARAM: plugin_id (str) - The ID of the plugin to run.\n # @PARAM: params (Dict[str, Any]) - Parameters for the plugin.\n # @PARAM: user_id (Optional[str]) - ID of the user requesting the task.\n # @RETURN: Task - The created task instance.\n # @THROWS: ValueError if plugin not found or params invalid.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n async def create_task(\n self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None\n ) -> Task:\n with belief_scope(\"TaskManager.create_task\", f\"plugin_id={plugin_id}\"):\n if not self.plugin_loader.has_plugin(plugin_id):\n log.explore(\"Plugin not found\", error=f\"Plugin with ID '{plugin_id}' not found.\")\n raise ValueError(f\"Plugin with ID '{plugin_id}' not found.\")\n\n self.plugin_loader.get_plugin(plugin_id)\n\n if not isinstance(params, dict):\n log.explore(\"Invalid task params type\", error=\"Task parameters must be a dictionary.\")\n raise ValueError(\"Task parameters must be a dictionary.\")\n\n log.reason(\"Creating task node and scheduling execution\", payload={\"plugin_id\": plugin_id})\n task = Task(plugin_id=plugin_id, params=params, user_id=user_id)\n self.tasks[task.id] = task\n self.persistence_service.persist_task(task)\n trace_id = get_trace_id()\n self.loop.create_task(\n self._run_task(task.id, trace_id=trace_id)\n ) # Schedule task for execution\n log.reflect(\n \"Task creation persisted and execution scheduled\",\n payload={\"task_id\": task.id, \"plugin_id\": plugin_id},\n )\n return task\n\n # [/DEF:create_task:Function]\n" }, { "contract_id": "_run_task", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 268, - "end_line": 365, + "start_line": 274, + "end_line": 367, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43720,14 +44476,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_run_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Internal method to execute a task with TaskContext support.\n # @PRE: Task exists in registry.\n # @POST: Task is executed, status updated to SUCCESS or FAILED.\n # @PARAM: task_id (str) - The ID of the task to run.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def _run_task(self, task_id: str):\n with belief_scope(\"TaskManager._run_task\", f\"task_id={task_id}\"):\n task = self.tasks[task_id]\n plugin = self.plugin_loader.get_plugin(task.plugin_id)\n\n logger.reason(\n \"Transitioning task to running state\",\n extra={\"task_id\": task_id, \"plugin_id\": task.plugin_id},\n )\n logger.info(\n f\"Starting execution of task {task_id} for plugin '{plugin.name}'\"\n )\n task.status = TaskStatus.RUNNING\n task.started_at = datetime.utcnow()\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task started for plugin '{plugin.name}'\",\n source=\"system\",\n )\n\n try:\n # Prepare params and check if plugin supports new TaskContext\n params = {**task.params, \"_task_id\": task_id}\n\n # Check if plugin's execute method accepts 'context' parameter\n sig = inspect.signature(plugin.execute)\n accepts_context = \"context\" in sig.parameters\n\n if accepts_context:\n # Create TaskContext for new-style plugins\n context = TaskContext(\n task_id=task_id,\n add_log_fn=self._add_log,\n params=params,\n default_source=\"plugin\",\n background_tasks=None,\n )\n\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params, context=context)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor,\n lambda: plugin.execute(params, context=context),\n )\n else:\n # Backward compatibility: old-style plugins without context\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor, plugin.execute, params\n )\n\n logger.info(f\"Task {task_id} completed successfully\")\n task.status = TaskStatus.SUCCESS\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task completed successfully for plugin '{plugin.name}'\",\n source=\"system\",\n )\n except Exception as e:\n logger.error(f\"Task {task_id} failed: {e}\")\n task.status = TaskStatus.FAILED\n self._add_log(\n task_id,\n \"ERROR\",\n f\"Task failed: {e}\",\n source=\"system\",\n metadata={\"error_type\": type(e).__name__},\n )\n finally:\n task.finished_at = datetime.utcnow()\n # Flush any remaining buffered logs before persisting task\n self._flush_task_logs(task_id)\n self.persistence_service.persist_task(task)\n logger.info(\n f\"Task {task_id} execution finished with status: {task.status}\"\n )\n logger.reflect(\n \"Task lifecycle reached persisted terminal state\",\n extra={\"task_id\": task_id, \"status\": str(task.status)},\n )\n\n # [/DEF:_run_task:Function]\n" + "body": " # [DEF:_run_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Internal method to execute a task with TaskContext support.\n # @PRE: Task exists in registry.\n # @POST: Task is executed, status updated to SUCCESS or FAILED.\n # @PARAM: task_id (str) - The ID of the task to run.\n # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]\n # @RELATION: [DEPENDS_ON] ->[JobLifecycle]\n # @RELATION: [DEPENDS_ON] ->[TaskContext]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n async def _run_task(self, task_id: str, trace_id: str = None):\n if trace_id:\n set_trace_id(trace_id)\n with belief_scope(\"TaskManager._run_task\", f\"task_id={task_id}\"):\n task = self.tasks[task_id]\n plugin = self.plugin_loader.get_plugin(task.plugin_id)\n\n log.reason(\n \"Transitioning task to running state\",\n payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id},\n )\n task.status = TaskStatus.RUNNING\n task.started_at = datetime.utcnow()\n self.persistence_service.persist_task(task)\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task started for plugin '{plugin.name}'\",\n source=\"system\",\n )\n\n try:\n # Prepare params and check if plugin supports new TaskContext\n params = {**task.params, \"_task_id\": task_id}\n\n # Check if plugin's execute method accepts 'context' parameter\n sig = inspect.signature(plugin.execute)\n accepts_context = \"context\" in sig.parameters\n\n if accepts_context:\n # Create TaskContext for new-style plugins\n context = TaskContext(\n task_id=task_id,\n add_log_fn=self._add_log,\n params=params,\n default_source=\"plugin\",\n background_tasks=None,\n )\n\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params, context=context)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor,\n lambda: plugin.execute(params, context=context),\n )\n else:\n # Backward compatibility: old-style plugins without context\n if asyncio.iscoroutinefunction(plugin.execute):\n task.result = await plugin.execute(params)\n else:\n task.result = await self.loop.run_in_executor(\n self.executor, plugin.execute, params\n )\n\n task.status = TaskStatus.SUCCESS\n self._add_log(\n task_id,\n \"INFO\",\n f\"Task completed successfully for plugin '{plugin.name}'\",\n source=\"system\",\n )\n log.reflect(\"Task completed successfully\", payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id})\n except Exception as e:\n task.status = TaskStatus.FAILED\n self._add_log(\n task_id,\n \"ERROR\",\n f\"Task failed: {e}\",\n source=\"system\",\n metadata={\"error_type\": type(e).__name__},\n )\n log.explore(\"Task failed during execution\", error=str(e), payload={\"task_id\": task_id, \"plugin_id\": task.plugin_id})\n finally:\n task.finished_at = datetime.utcnow()\n # Flush any remaining buffered logs before persisting task\n self._flush_task_logs(task_id)\n self.persistence_service.persist_task(task)\n log.reflect(\n \"Task execution finished\",\n payload={\"task_id\": task_id, \"status\": str(task.status)},\n )\n\n # [/DEF:_run_task:Function]\n" }, { "contract_id": "resolve_task", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 367, - "end_line": 393, + "start_line": 369, + "end_line": 395, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43825,8 +44581,8 @@ "contract_id": "wait_for_resolution", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 395, - "end_line": 419, + "start_line": 397, + "end_line": 421, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43917,8 +44673,8 @@ "contract_id": "wait_for_input", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 421, - "end_line": 443, + "start_line": 423, + "end_line": 445, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -43986,8 +44742,8 @@ "contract_id": "get_task", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 445, - "end_line": 457, + "start_line": 447, + "end_line": 459, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44062,8 +44818,8 @@ "contract_id": "get_all_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 459, - "end_line": 470, + "start_line": 461, + "end_line": 472, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44131,8 +44887,8 @@ "contract_id": "get_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 472, - "end_line": 516, + "start_line": 474, + "end_line": 518, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44207,8 +44963,8 @@ "contract_id": "get_task_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 518, - "end_line": 554, + "start_line": 520, + "end_line": 556, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44306,8 +45062,8 @@ "contract_id": "get_task_log_stats", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 556, - "end_line": 569, + "start_line": 558, + "end_line": 571, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44405,8 +45161,8 @@ "contract_id": "get_task_log_sources", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 571, - "end_line": 584, + "start_line": 573, + "end_line": 586, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44504,8 +45260,8 @@ "contract_id": "_add_log", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 586, - "end_line": 640, + "start_line": 588, + "end_line": 642, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44596,8 +45352,8 @@ "contract_id": "subscribe_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 642, - "end_line": 658, + "start_line": 644, + "end_line": 660, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44672,8 +45428,8 @@ "contract_id": "unsubscribe_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 660, - "end_line": 676, + "start_line": 662, + "end_line": 678, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44741,8 +45497,8 @@ "contract_id": "load_persisted_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 678, - "end_line": 692, + "start_line": 680, + "end_line": 694, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44826,8 +45582,8 @@ "contract_id": "await_input", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 694, - "end_line": 725, + "start_line": 696, + "end_line": 727, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -44925,8 +45681,8 @@ "contract_id": "resume_task_with_password", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 727, - "end_line": 769, + "start_line": 729, + "end_line": 771, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -45024,8 +45780,8 @@ "contract_id": "clear_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/manager.py", - "start_line": 771, - "end_line": 824, + "start_line": 773, + "end_line": 826, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -45140,7 +45896,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:clear_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Clears tasks based on status filter (also deletes associated logs).\n # @PRE: status is Optional[TaskStatus].\n # @POST: Tasks matching filter (or all non-active) cleared from registry and database.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: int - Number of tasks cleared.\n # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:\n with belief_scope(\"TaskManager.clear_tasks\"):\n tasks_to_remove = []\n for task_id, task in list(self.tasks.items()):\n # If status is provided, match it.\n # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)\n # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.\n # RUNNING is active execution.\n\n should_remove = False\n if status:\n if task.status == status:\n should_remove = True\n else:\n # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)\n if task.status not in [\n TaskStatus.RUNNING,\n TaskStatus.AWAITING_INPUT,\n TaskStatus.AWAITING_MAPPING,\n ]:\n should_remove = True\n\n if should_remove:\n tasks_to_remove.append(task_id)\n\n for tid in tasks_to_remove:\n # Cancel future if exists (e.g. for AWAITING_INPUT/MAPPING)\n if tid in self.task_futures:\n self.task_futures[tid].cancel()\n del self.task_futures[tid]\n\n del self.tasks[tid]\n\n # Remove from persistence (task_records and task_logs via CASCADE)\n self.persistence_service.delete_tasks(tasks_to_remove)\n\n # Also explicitly delete logs (in case CASCADE is not set up)\n if tasks_to_remove:\n self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)\n\n logger.info(f\"Cleared {len(tasks_to_remove)} tasks.\")\n return len(tasks_to_remove)\n\n # [/DEF:clear_tasks:Function]\n" + "body": " # [DEF:clear_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Clears tasks based on status filter (also deletes associated logs).\n # @PRE: status is Optional[TaskStatus].\n # @POST: Tasks matching filter (or all non-active) cleared from registry and database.\n # @PARAM: status (Optional[TaskStatus]) - Filter by task status.\n # @RETURN: int - Number of tasks cleared.\n # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]\n # @RELATION: [DEPENDS_ON] ->[TaskGraph]\n # @RELATION: [DEPENDS_ON] ->[EventBus]\n def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:\n with belief_scope(\"TaskManager.clear_tasks\"):\n tasks_to_remove = []\n for task_id, task in list(self.tasks.items()):\n # If status is provided, match it.\n # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)\n # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.\n # RUNNING is active execution.\n\n should_remove = False\n if status:\n if task.status == status:\n should_remove = True\n else:\n # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)\n if task.status not in [\n TaskStatus.RUNNING,\n TaskStatus.AWAITING_INPUT,\n TaskStatus.AWAITING_MAPPING,\n ]:\n should_remove = True\n\n if should_remove:\n tasks_to_remove.append(task_id)\n\n for tid in tasks_to_remove:\n # Cancel future if exists (e.g. for AWAITING_INPUT/MAPPING)\n if tid in self.task_futures:\n self.task_futures[tid].cancel()\n del self.task_futures[tid]\n\n del self.tasks[tid]\n\n # Remove from persistence (task_records and task_logs via CASCADE)\n self.persistence_service.delete_tasks(tasks_to_remove)\n\n # Also explicitly delete logs (in case CASCADE is not set up)\n if tasks_to_remove:\n self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)\n\n log.reflect(\"Tasks cleared from registry and database\", payload={\"count\": len(tasks_to_remove)})\n return len(tasks_to_remove)\n\n # [/DEF:clear_tasks:Function]\n" }, { "contract_id": "TaskManagerModels", @@ -45563,7 +46319,7 @@ "contract_type": "Module", "file_path": "backend/src/core/task_manager/persistence.py", "start_line": 1, - "end_line": 624, + "end_line": 655, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -45671,14 +46427,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:TaskPersistenceModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, sqlite, sqlalchemy, task, storage\n# @PURPOSE: Handles the persistence of tasks using SQLAlchemy and the tasks.db database.\n# @LAYER: Core\n# @PRE: Tasks database must be initialized with TaskRecord and TaskLogRecord schemas.\n# @POST: Provides reliable storage and retrieval for task metadata and logs.\n# @SIDE_EFFECT: Performs database I/O on tasks.db.\n# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @INVARIANT: Database schema must match the TaskRecord model structure.\n\n# [SECTION: IMPORTS]\nfrom datetime import datetime\nfrom typing import List, Optional\nimport json\nimport re\n\nfrom sqlalchemy.orm import Session\nfrom ...models.task import TaskRecord, TaskLogRecord\nfrom ...models.mapping import Environment\nfrom ..database import TasksSessionLocal\nfrom .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats\nfrom ..logger import logger, belief_scope\n# [/SECTION]\n\n\n# [DEF:TaskPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, sqlalchemy\n# @PURPOSE: Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.\n# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskRecord]\n# @RELATION: [DEPENDS_ON] ->[Environment]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @INVARIANT: Persistence must handle potentially missing task fields natively.\n#\n# @TEST_CONTRACT: TaskPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"persist_task creates or updates a record\",\n# \"load_tasks retrieves valid Task instances\",\n# \"delete_tasks correctly removes records from the database\"\n# ]\n# }\n# @TEST_FIXTURE: valid_task_persistence -> {\"task_id\": \"123\", \"status\": \"PENDING\"}\n# @TEST_EDGE: persist_invalid_task_type -> raises Exception\n# @TEST_EDGE: load_corrupt_json_params -> handled gracefully\n# @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]\nclass TaskPersistenceService:\n # [DEF:_json_load_if_needed:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely load JSON strings from DB if necessary\n # @PRE: value is an arbitrary database value\n # @POST: Returns parsed JSON object, list, string, or primitive\n @staticmethod\n def _json_load_if_needed(value):\n with belief_scope(\"TaskPersistenceService._json_load_if_needed\"):\n if value is None:\n return None\n if isinstance(value, (dict, list)):\n return value\n if isinstance(value, str):\n stripped = value.strip()\n if stripped == \"\" or stripped.lower() == \"null\":\n return None\n try:\n return json.loads(stripped)\n except json.JSONDecodeError:\n return value\n return value\n\n # [/DEF:_json_load_if_needed:Function]\n\n # [DEF:_parse_datetime:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely parse a datetime string from the database\n # @PRE: value is an ISO string or datetime object\n # @POST: Returns datetime object or None\n @staticmethod\n def _parse_datetime(value):\n with belief_scope(\"TaskPersistenceService._parse_datetime\"):\n if value is None or isinstance(value, datetime):\n return value\n if isinstance(value, str):\n try:\n return datetime.fromisoformat(value)\n except ValueError:\n return None\n return None\n\n # [/DEF:_parse_datetime:Function]\n\n # [DEF:_resolve_environment_id:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints.\n # @PRE: Session is active\n # @POST: Returns existing environments.id or None when unresolved.\n # @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]\n # @RELATION: [DEPENDS_ON] ->[Environment]\n @staticmethod\n def _resolve_environment_id(\n session: Session, env_id: Optional[str]\n ) -> Optional[str]:\n with belief_scope(\"_resolve_environment_id\"):\n raw_value = str(env_id or \"\").strip()\n if not raw_value:\n return None\n\n # 1) Direct match by primary key.\n by_id = (\n session.query(Environment).filter(Environment.id == raw_value).first()\n )\n if by_id:\n return str(by_id.id)\n\n # 2) Exact match by name.\n by_name = (\n session.query(Environment).filter(Environment.name == raw_value).first()\n )\n if by_name:\n return str(by_name.id)\n\n # 3) Slug-like match (e.g. \"ss-dev\" -> \"SS DEV\").\n def normalize_token(value: str) -> str:\n lowered = str(value or \"\").strip().lower()\n return re.sub(r\"[^a-z0-9]+\", \"-\", lowered).strip(\"-\")\n\n target_token = normalize_token(raw_value)\n if not target_token:\n return None\n\n for env in session.query(Environment).all():\n if (\n normalize_token(env.id) == target_token\n or normalize_token(env.name) == target_token\n ):\n return str(env.id)\n\n return None\n\n # [/DEF:_resolve_environment_id:Function]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the persistence service.\n # @PRE: None.\n # @POST: Service is ready.\n def __init__(self):\n with belief_scope(\"TaskPersistenceService.__init__\"):\n # We use TasksSessionLocal from database.py\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:persist_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists or updates a single task in the database.\n # @PRE: isinstance(task, Task)\n # @POST: Task record created or updated in database.\n # @PARAM: task (Task) - The task object to persist.\n # @SIDE_EFFECT: Writes to task_records table in tasks.db\n # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]\n # @RELATION: [CALLS] ->[_resolve_environment_id]\n def persist_task(self, task: Task) -> None:\n with belief_scope(\"TaskPersistenceService.persist_task\", f\"task_id={task.id}\"):\n session: Session = TasksSessionLocal()\n try:\n record = (\n session.query(TaskRecord).filter(TaskRecord.id == task.id).first()\n )\n if not record:\n record = TaskRecord(id=task.id)\n session.add(record)\n\n record.type = task.plugin_id\n record.status = task.status.value\n raw_env_id = task.params.get(\"environment_id\") or task.params.get(\n \"source_env_id\"\n )\n record.environment_id = self._resolve_environment_id(\n session, raw_env_id\n )\n record.started_at = task.started_at\n record.finished_at = task.finished_at\n\n # Ensure params and result are JSON serializable\n def json_serializable(obj):\n with belief_scope(\"TaskPersistenceService.json_serializable\"):\n if isinstance(obj, dict):\n return {k: json_serializable(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [json_serializable(v) for v in obj]\n elif isinstance(obj, datetime):\n return obj.isoformat()\n return obj\n\n record.params = json_serializable(task.params)\n record.result = json_serializable(task.result)\n\n # Store logs as JSON, converting datetime to string\n record.logs = []\n for log in task.logs:\n log_dict = log.dict()\n if isinstance(log_dict.get(\"timestamp\"), datetime):\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n # Also clean up any datetimes in context\n if log_dict.get(\"context\"):\n log_dict[\"context\"] = json_serializable(log_dict[\"context\"])\n record.logs.append(log_dict)\n\n # Extract error if failed\n if task.status == TaskStatus.FAILED:\n for log in reversed(task.logs):\n if log.level == \"ERROR\":\n record.error = log.message\n break\n\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to persist task {task.id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:persist_task:Function]\n\n # [DEF:persist_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists multiple tasks.\n # @PRE: isinstance(tasks, list)\n # @POST: All tasks in list are persisted.\n # @PARAM: tasks (List[Task]) - The list of tasks to persist.\n # @RELATION: [CALLS] ->[persist_task]\n def persist_tasks(self, tasks: List[Task]) -> None:\n with belief_scope(\"TaskPersistenceService.persist_tasks\"):\n for task in tasks:\n self.persist_task(task)\n\n # [/DEF:persist_tasks:Function]\n\n # [DEF:load_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Loads tasks from the database.\n # @PRE: limit is an integer.\n # @POST: Returns list of Task objects.\n # @PARAM: limit (int) - Max tasks to load.\n # @PARAM: status (Optional[TaskStatus]) - Filter by status.\n # @RETURN: List[Task] - The loaded tasks.\n # @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]\n # @RELATION: [CALLS] ->[_json_load_if_needed]\n # @RELATION: [CALLS] ->[_parse_datetime]\n def load_tasks(\n self, limit: int = 100, status: Optional[TaskStatus] = None\n ) -> List[Task]:\n with belief_scope(\"TaskPersistenceService.load_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskRecord)\n if status:\n query = query.filter(TaskRecord.status == status.value)\n\n records = (\n query.order_by(TaskRecord.created_at.desc()).limit(limit).all()\n )\n\n loaded_tasks = []\n for record in records:\n try:\n logs = []\n logs_payload = self._json_load_if_needed(record.logs)\n if isinstance(logs_payload, list):\n for log_data in logs_payload:\n if not isinstance(log_data, dict):\n continue\n log_data = dict(log_data)\n log_data[\"timestamp\"] = (\n self._parse_datetime(log_data.get(\"timestamp\"))\n or datetime.utcnow()\n )\n logs.append(LogEntry(**log_data))\n\n started_at = self._parse_datetime(record.started_at)\n finished_at = self._parse_datetime(record.finished_at)\n params = self._json_load_if_needed(record.params)\n result = self._json_load_if_needed(record.result)\n\n task = Task(\n id=record.id,\n plugin_id=record.type,\n status=TaskStatus(record.status),\n started_at=started_at,\n finished_at=finished_at,\n params=params if isinstance(params, dict) else {},\n result=result,\n logs=logs,\n )\n loaded_tasks.append(task)\n except Exception as e:\n logger.error(f\"Failed to reconstruct task {record.id}: {e}\")\n\n return loaded_tasks\n finally:\n session.close()\n\n # [/DEF:load_tasks:Function]\n\n # [DEF:delete_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Deletes specific tasks from the database.\n # @PRE: task_ids is a list of strings.\n # @POST: Specified task records deleted from database.\n # @PARAM: task_ids (List[str]) - List of task IDs to delete.\n # @SIDE_EFFECT: Deletes rows from task_records table.\n # @RELATION: [DEPENDS_ON] ->[TaskRecord]\n def delete_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskPersistenceService.delete_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(\n synchronize_session=False\n )\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_tasks:Function]\n\n\n# [/DEF:TaskPersistenceService:Class]\n\n\n# [DEF:TaskLogPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, log, sqlalchemy\n# @PURPOSE: Provides methods to store, query, summarize, and delete task log rows in the task_logs table.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.\n# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]\n# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @INVARIANT: Log entries are batch-inserted for performance.\n#\n# @TEST_CONTRACT: TaskLogPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"add_logs efficiently saves logs to the database\",\n# \"get_logs retrieves properly filtered LogEntry objects\"\n# ]\n# }\n# @TEST_FIXTURE: valid_log_batch -> {\"task_id\": \"123\", \"logs\": [{\"level\": \"INFO\", \"message\": \"msg\"}]}\n# @TEST_EDGE: empty_log_list -> no-op behavior\n# @TEST_EDGE: add_logs_db_error -> rollback and log error\n# @TEST_INVARIANT: accurate_log_aggregation -> verifies: [valid_log_batch]\nclass TaskLogPersistenceService:\n \"\"\"\n Service for persisting and querying task logs.\n Supports batch inserts, filtering, and statistics.\n \"\"\"\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the TaskLogPersistenceService\n # @PRE: config is provided or defaults are used\n # @POST: Service is ready for log persistence\n def __init__(self, config=None):\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:add_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Batch insert log entries for a task.\n # @PRE: logs is a list of LogEntry objects.\n # @POST: All logs inserted into task_logs table.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: logs (List[LogEntry]) - Log entries to insert.\n # @SIDE_EFFECT: Writes to task_logs table.\n # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:\n if not logs:\n return\n with belief_scope(\"TaskLogPersistenceService.add_logs\", f\"task_id={task_id}\"):\n session: Session = TasksSessionLocal()\n try:\n for log in logs:\n record = TaskLogRecord(\n task_id=task_id,\n timestamp=log.timestamp,\n level=log.level,\n source=log.source or \"system\",\n message=log.message,\n metadata_json=json.dumps(log.metadata)\n if log.metadata\n else None,\n )\n session.add(record)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to add logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:add_logs:Function]\n\n # [DEF:get_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Query logs for a task with filtering and pagination.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of TaskLog objects matching filters.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: log_filter (LogFilter) - Filter parameters.\n # @RETURN: List[TaskLog] - Filtered log entries.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogFilter]\n # @RELATION: [DEPENDS_ON] ->[TaskLog]\n def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:\n with belief_scope(\"TaskLogPersistenceService.get_logs\", f\"task_id={task_id}\"):\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n )\n\n # Apply filters\n if log_filter.level:\n query = query.filter(\n TaskLogRecord.level == log_filter.level.upper()\n )\n if log_filter.source:\n query = query.filter(TaskLogRecord.source == log_filter.source)\n if log_filter.search:\n search_pattern = f\"%{log_filter.search}%\"\n query = query.filter(TaskLogRecord.message.ilike(search_pattern))\n\n # Order by timestamp ascending (oldest first)\n query = query.order_by(TaskLogRecord.timestamp.asc())\n\n # Apply pagination\n records = query.offset(log_filter.offset).limit(log_filter.limit).all()\n\n logs = []\n for record in records:\n metadata = None\n if record.metadata_json:\n try:\n metadata = json.loads(record.metadata_json)\n except json.JSONDecodeError:\n metadata = None\n\n logs.append(\n TaskLog(\n id=record.id,\n task_id=record.task_id,\n timestamp=record.timestamp,\n level=record.level,\n source=record.source,\n message=record.message,\n metadata=metadata,\n )\n )\n\n return logs\n finally:\n session.close()\n\n # [/DEF:get_logs:Function]\n\n # [DEF:get_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogStats]\n def get_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\n \"TaskLogPersistenceService.get_log_stats\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n # Get total count\n total_count = (\n session.query(TaskLogRecord)\n .filter(TaskLogRecord.task_id == task_id)\n .count()\n )\n\n # Get counts by level\n from sqlalchemy import func\n\n level_counts = (\n session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.level)\n .all()\n )\n\n by_level = {level: count for level, count in level_counts}\n\n # Get counts by source\n source_counts = (\n session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.source)\n .all()\n )\n\n by_source = {source: count for source, count in source_counts}\n\n return LogStats(\n total_count=total_count, by_level=by_level, by_source=by_source\n )\n finally:\n session.close()\n\n # [/DEF:get_log_stats:Function]\n\n # [DEF:get_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def get_sources(self, task_id: str) -> List[str]:\n with belief_scope(\n \"TaskLogPersistenceService.get_sources\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n from sqlalchemy import distinct\n\n sources = (\n session.query(distinct(TaskLogRecord.source))\n .filter(TaskLogRecord.task_id == task_id)\n .all()\n )\n return [s[0] for s in sources]\n finally:\n session.close()\n\n # [/DEF:get_sources:Function]\n\n # [DEF:delete_logs_for_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for a specific task.\n # @PRE: task_id is a valid task ID.\n # @POST: All logs for the task are deleted.\n # @PARAM: task_id (str) - The task ID.\n # @SIDE_EFFECT: Deletes from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_task(self, task_id: str) -> None:\n with belief_scope(\n \"TaskLogPersistenceService.delete_logs_for_task\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n ).delete(synchronize_session=False)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_task:Function]\n\n # [DEF:delete_logs_for_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for multiple tasks.\n # @PRE: task_ids is a list of task IDs.\n # @POST: All logs for the tasks are deleted.\n # @PARAM: task_ids (List[str]) - List of task IDs.\n # @SIDE_EFFECT: Deletes rows from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskLogPersistenceService.delete_logs_for_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id.in_(task_ids)\n ).delete(synchronize_session=False)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete logs for tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_tasks:Function]\n\n\n# [/DEF:TaskLogPersistenceService:Class]\n# [/DEF:TaskPersistenceModule:Module]\n" + "body": "# [DEF:TaskPersistenceModule:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, sqlite, sqlalchemy, task, storage\n# @PURPOSE: Handles the persistence of tasks using SQLAlchemy and the tasks.db database.\n# @LAYER: Core\n# @PRE: Tasks database must be initialized with TaskRecord and TaskLogRecord schemas.\n# @POST: Provides reliable storage and retrieval for task metadata and logs.\n# @SIDE_EFFECT: Performs database I/O on tasks.db.\n# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @INVARIANT: Database schema must match the TaskRecord model structure.\n\n# [SECTION: IMPORTS]\nfrom datetime import datetime\nfrom typing import List, Optional\nimport json\nimport re\n\nfrom sqlalchemy.orm import Session\nfrom ...models.task import TaskRecord, TaskLogRecord\nfrom ...models.mapping import Environment\nfrom ..database import TasksSessionLocal\nfrom .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats\nfrom ..logger import logger, belief_scope\nfrom ..cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"TaskPersistence\")\nlog_pl = MarkerLogger(\"TaskLogPersistence\")\n# [/SECTION]\n\n\n# [DEF:TaskPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, sqlalchemy\n# @PURPOSE: Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.\n# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskRecord]\n# @RELATION: [DEPENDS_ON] ->[Environment]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @INVARIANT: Persistence must handle potentially missing task fields natively.\n#\n# @TEST_CONTRACT: TaskPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"persist_task creates or updates a record\",\n# \"load_tasks retrieves valid Task instances\",\n# \"delete_tasks correctly removes records from the database\"\n# ]\n# }\n# @TEST_FIXTURE: valid_task_persistence -> {\"task_id\": \"123\", \"status\": \"PENDING\"}\n# @TEST_EDGE: persist_invalid_task_type -> raises Exception\n# @TEST_EDGE: load_corrupt_json_params -> handled gracefully\n# @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]\nclass TaskPersistenceService:\n # [DEF:_json_load_if_needed:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely load JSON strings from DB if necessary\n # @PRE: value is an arbitrary database value\n # @POST: Returns parsed JSON object, list, string, or primitive\n @staticmethod\n def _json_load_if_needed(value):\n with belief_scope(\"TaskPersistenceService._json_load_if_needed\"):\n if value is None:\n return None\n if isinstance(value, (dict, list)):\n return value\n if isinstance(value, str):\n stripped = value.strip()\n if stripped == \"\" or stripped.lower() == \"null\":\n return None\n try:\n return json.loads(stripped)\n except json.JSONDecodeError:\n return value\n return value\n\n # [/DEF:_json_load_if_needed:Function]\n\n # [DEF:_parse_datetime:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely parse a datetime string from the database\n # @PRE: value is an ISO string or datetime object\n # @POST: Returns datetime object or None\n @staticmethod\n def _parse_datetime(value):\n with belief_scope(\"TaskPersistenceService._parse_datetime\"):\n if value is None or isinstance(value, datetime):\n return value\n if isinstance(value, str):\n try:\n return datetime.fromisoformat(value)\n except ValueError:\n return None\n return None\n\n # [/DEF:_parse_datetime:Function]\n\n # [DEF:_resolve_environment_id:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints.\n # @PRE: Session is active\n # @POST: Returns existing environments.id or None when unresolved.\n # @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]\n # @RELATION: [DEPENDS_ON] ->[Environment]\n @staticmethod\n def _resolve_environment_id(\n session: Session, env_id: Optional[str]\n ) -> Optional[str]:\n with belief_scope(\"_resolve_environment_id\"):\n raw_value = str(env_id or \"\").strip()\n if not raw_value:\n return None\n\n # 1) Direct match by primary key.\n by_id = (\n session.query(Environment).filter(Environment.id == raw_value).first()\n )\n if by_id:\n return str(by_id.id)\n\n # 2) Exact match by name.\n by_name = (\n session.query(Environment).filter(Environment.name == raw_value).first()\n )\n if by_name:\n return str(by_name.id)\n\n # 3) Slug-like match (e.g. \"ss-dev\" -> \"SS DEV\").\n def normalize_token(value: str) -> str:\n lowered = str(value or \"\").strip().lower()\n return re.sub(r\"[^a-z0-9]+\", \"-\", lowered).strip(\"-\")\n\n target_token = normalize_token(raw_value)\n if not target_token:\n return None\n\n for env in session.query(Environment).all():\n if (\n normalize_token(env.id) == target_token\n or normalize_token(env.name) == target_token\n ):\n return str(env.id)\n\n return None\n\n # [/DEF:_resolve_environment_id:Function]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the persistence service.\n # @PRE: None.\n # @POST: Service is ready.\n def __init__(self):\n with belief_scope(\"TaskPersistenceService.__init__\"):\n # We use TasksSessionLocal from database.py\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:persist_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists or updates a single task in the database.\n # @PRE: isinstance(task, Task)\n # @POST: Task record created or updated in database.\n # @PARAM: task (Task) - The task object to persist.\n # @SIDE_EFFECT: Writes to task_records table in tasks.db\n # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]\n # @RELATION: [CALLS] ->[_resolve_environment_id]\n def persist_task(self, task: Task) -> None:\n with belief_scope(\"TaskPersistenceService.persist_task\", f\"task_id={task.id}\"):\n log.reason(\"Persisting task to database\", payload={\"task_id\": task.id})\n session: Session = TasksSessionLocal()\n try:\n record = (\n session.query(TaskRecord).filter(TaskRecord.id == task.id).first()\n )\n if not record:\n record = TaskRecord(id=task.id)\n session.add(record)\n\n record.type = task.plugin_id\n record.status = task.status.value\n raw_env_id = task.params.get(\"environment_id\") or task.params.get(\n \"source_env_id\"\n )\n record.environment_id = self._resolve_environment_id(\n session, raw_env_id\n )\n record.started_at = task.started_at\n record.finished_at = task.finished_at\n\n # Ensure params and result are JSON serializable\n def json_serializable(obj):\n with belief_scope(\"TaskPersistenceService.json_serializable\"):\n if isinstance(obj, dict):\n return {k: json_serializable(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [json_serializable(v) for v in obj]\n elif isinstance(obj, datetime):\n return obj.isoformat()\n return obj\n\n record.params = json_serializable(task.params)\n record.result = json_serializable(task.result)\n\n # Store logs as JSON, converting datetime to string\n record.logs = []\n for log_entry in task.logs:\n log_dict = log_entry.dict()\n if isinstance(log_dict.get(\"timestamp\"), datetime):\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n # Also clean up any datetimes in context\n if log_dict.get(\"context\"):\n log_dict[\"context\"] = json_serializable(log_dict[\"context\"])\n record.logs.append(log_dict)\n\n # Extract error if failed\n if task.status == TaskStatus.FAILED:\n for log_entry in reversed(task.logs):\n if log_entry.level == \"ERROR\":\n record.error = log_entry.message\n break\n\n session.commit()\n log.reflect(\"Task persisted successfully\", payload={\"task_id\": task.id})\n except Exception as e:\n session.rollback()\n log.explore(\"Failed to persist task\", error=str(e), payload={\"task_id\": task.id})\n logger.error(f\"Failed to persist task {task.id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:persist_task:Function]\n\n # [DEF:persist_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists multiple tasks.\n # @PRE: isinstance(tasks, list)\n # @POST: All tasks in list are persisted.\n # @PARAM: tasks (List[Task]) - The list of tasks to persist.\n # @RELATION: [CALLS] ->[persist_task]\n def persist_tasks(self, tasks: List[Task]) -> None:\n with belief_scope(\"TaskPersistenceService.persist_tasks\"):\n log.reason(\"Persisting multiple tasks\", payload={\"count\": len(tasks)})\n for task in tasks:\n self.persist_task(task)\n log.reflect(\"All tasks persisted\", payload={\"count\": len(tasks)})\n\n # [/DEF:persist_tasks:Function]\n\n # [DEF:load_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Loads tasks from the database.\n # @PRE: limit is an integer.\n # @POST: Returns list of Task objects.\n # @PARAM: limit (int) - Max tasks to load.\n # @PARAM: status (Optional[TaskStatus]) - Filter by status.\n # @RETURN: List[Task] - The loaded tasks.\n # @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]\n # @RELATION: [CALLS] ->[_json_load_if_needed]\n # @RELATION: [CALLS] ->[_parse_datetime]\n def load_tasks(\n self, limit: int = 100, status: Optional[TaskStatus] = None\n ) -> List[Task]:\n with belief_scope(\"TaskPersistenceService.load_tasks\"):\n log.reason(\"Loading tasks from database\", payload={\"limit\": limit, \"status\": str(status) if status else None})\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskRecord)\n if status:\n query = query.filter(TaskRecord.status == status.value)\n\n records = (\n query.order_by(TaskRecord.created_at.desc()).limit(limit).all()\n )\n\n loaded_tasks = []\n for record in records:\n try:\n logs = []\n logs_payload = self._json_load_if_needed(record.logs)\n if isinstance(logs_payload, list):\n for log_data in logs_payload:\n if not isinstance(log_data, dict):\n continue\n log_data = dict(log_data)\n log_data[\"timestamp\"] = (\n self._parse_datetime(log_data.get(\"timestamp\"))\n or datetime.utcnow()\n )\n logs.append(LogEntry(**log_data))\n\n started_at = self._parse_datetime(record.started_at)\n finished_at = self._parse_datetime(record.finished_at)\n params = self._json_load_if_needed(record.params)\n result = self._json_load_if_needed(record.result)\n\n task = Task(\n id=record.id,\n plugin_id=record.type,\n status=TaskStatus(record.status),\n started_at=started_at,\n finished_at=finished_at,\n params=params if isinstance(params, dict) else {},\n result=result,\n logs=logs,\n )\n loaded_tasks.append(task)\n except Exception as e:\n log.explore(\"Failed to reconstruct task from record\", error=str(e), payload={\"record_id\": record.id})\n logger.error(f\"Failed to reconstruct task {record.id}: {e}\")\n\n log.reflect(\"Tasks loaded from database\", payload={\"count\": len(loaded_tasks)})\n return loaded_tasks\n finally:\n session.close()\n\n # [/DEF:load_tasks:Function]\n\n # [DEF:delete_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Deletes specific tasks from the database.\n # @PRE: task_ids is a list of strings.\n # @POST: Specified task records deleted from database.\n # @PARAM: task_ids (List[str]) - List of task IDs to delete.\n # @SIDE_EFFECT: Deletes rows from task_records table.\n # @RELATION: [DEPENDS_ON] ->[TaskRecord]\n def delete_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskPersistenceService.delete_tasks\"):\n log.reason(\"Deleting tasks from database\", payload={\"count\": len(task_ids)})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(\n synchronize_session=False\n )\n session.commit()\n log.reflect(\"Tasks deleted from database\", payload={\"count\": len(task_ids)})\n except Exception as e:\n session.rollback()\n log.explore(\"Failed to delete tasks\", error=str(e))\n logger.error(f\"Failed to delete tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_tasks:Function]\n\n\n# [/DEF:TaskPersistenceService:Class]\n\n\n# [DEF:TaskLogPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, log, sqlalchemy\n# @PURPOSE: Provides methods to store, query, summarize, and delete task log rows in the task_logs table.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.\n# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]\n# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @INVARIANT: Log entries are batch-inserted for performance.\n#\n# @TEST_CONTRACT: TaskLogPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"add_logs efficiently saves logs to the database\",\n# \"get_logs retrieves properly filtered LogEntry objects\"\n# ]\n# }\n# @TEST_FIXTURE: valid_log_batch -> {\"task_id\": \"123\", \"logs\": [{\"level\": \"INFO\", \"message\": \"msg\"}]}\n# @TEST_EDGE: empty_log_list -> no-op behavior\n# @TEST_EDGE: add_logs_db_error -> rollback and log error\n# @TEST_INVARIANT: accurate_log_aggregation -> verifies: [valid_log_batch]\nclass TaskLogPersistenceService:\n \"\"\"\n Service for persisting and querying task logs.\n Supports batch inserts, filtering, and statistics.\n \"\"\"\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the TaskLogPersistenceService\n # @PRE: config is provided or defaults are used\n # @POST: Service is ready for log persistence\n def __init__(self, config=None):\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:add_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Batch insert log entries for a task.\n # @PRE: logs is a list of LogEntry objects.\n # @POST: All logs inserted into task_logs table.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: logs (List[LogEntry]) - Log entries to insert.\n # @SIDE_EFFECT: Writes to task_logs table.\n # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:\n if not logs:\n return\n with belief_scope(\"TaskLogPersistenceService.add_logs\", f\"task_id={task_id}\"):\n log_pl.reason(\"Batch inserting log entries\", payload={\"task_id\": task_id, \"count\": len(logs)})\n session: Session = TasksSessionLocal()\n try:\n for log_entry in logs:\n record = TaskLogRecord(\n task_id=task_id,\n timestamp=log_entry.timestamp,\n level=log_entry.level,\n source=log_entry.source or \"system\",\n message=log_entry.message,\n metadata_json=json.dumps(log_entry.metadata)\n if log_entry.metadata\n else None,\n )\n session.add(record)\n session.commit()\n log_pl.reflect(\"Log entries persisted\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to persist log entries\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to add logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:add_logs:Function]\n\n # [DEF:get_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Query logs for a task with filtering and pagination.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of TaskLog objects matching filters.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: log_filter (LogFilter) - Filter parameters.\n # @RETURN: List[TaskLog] - Filtered log entries.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogFilter]\n # @RELATION: [DEPENDS_ON] ->[TaskLog]\n def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:\n with belief_scope(\"TaskLogPersistenceService.get_logs\", f\"task_id={task_id}\"):\n log_pl.reason(\"Querying task logs\", payload={\"task_id\": task_id, \"filter_level\": log_filter.level})\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n )\n\n # Apply filters\n if log_filter.level:\n query = query.filter(\n TaskLogRecord.level == log_filter.level.upper()\n )\n if log_filter.source:\n query = query.filter(TaskLogRecord.source == log_filter.source)\n if log_filter.search:\n search_pattern = f\"%{log_filter.search}%\"\n query = query.filter(TaskLogRecord.message.ilike(search_pattern))\n\n # Order by timestamp ascending (oldest first)\n query = query.order_by(TaskLogRecord.timestamp.asc())\n\n # Apply pagination\n records = query.offset(log_filter.offset).limit(log_filter.limit).all()\n\n logs = []\n for record in records:\n metadata = None\n if record.metadata_json:\n try:\n metadata = json.loads(record.metadata_json)\n except json.JSONDecodeError:\n metadata = None\n\n logs.append(\n TaskLog(\n id=record.id,\n task_id=record.task_id,\n timestamp=record.timestamp,\n level=record.level,\n source=record.source,\n message=record.message,\n metadata=metadata,\n )\n )\n\n log_pl.reflect(\"Task logs retrieved\", payload={\"task_id\": task_id, \"count\": len(logs)})\n return logs\n finally:\n session.close()\n\n # [/DEF:get_logs:Function]\n\n # [DEF:get_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogStats]\n def get_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\n \"TaskLogPersistenceService.get_log_stats\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Aggregating log statistics\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n # Get total count\n total_count = (\n session.query(TaskLogRecord)\n .filter(TaskLogRecord.task_id == task_id)\n .count()\n )\n\n # Get counts by level\n from sqlalchemy import func\n\n level_counts = (\n session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.level)\n .all()\n )\n\n by_level = {level: count for level, count in level_counts}\n\n # Get counts by source\n source_counts = (\n session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.source)\n .all()\n )\n\n by_source = {source: count for source, count in source_counts}\n\n log_pl.reflect(\"Log statistics computed\", payload={\"task_id\": task_id, \"total\": total_count})\n return LogStats(\n total_count=total_count, by_level=by_level, by_source=by_source\n )\n finally:\n session.close()\n\n # [/DEF:get_log_stats:Function]\n\n # [DEF:get_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def get_sources(self, task_id: str) -> List[str]:\n with belief_scope(\n \"TaskLogPersistenceService.get_sources\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Querying distinct log sources\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n from sqlalchemy import distinct\n\n sources = (\n session.query(distinct(TaskLogRecord.source))\n .filter(TaskLogRecord.task_id == task_id)\n .all()\n )\n result = [s[0] for s in sources]\n log_pl.reflect(\"Log sources retrieved\", payload={\"task_id\": task_id, \"sources\": result})\n return result\n finally:\n session.close()\n\n # [/DEF:get_sources:Function]\n\n # [DEF:delete_logs_for_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for a specific task.\n # @PRE: task_id is a valid task ID.\n # @POST: All logs for the task are deleted.\n # @PARAM: task_id (str) - The task ID.\n # @SIDE_EFFECT: Deletes from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_task(self, task_id: str) -> None:\n with belief_scope(\n \"TaskLogPersistenceService.delete_logs_for_task\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Deleting logs for task\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n ).delete(synchronize_session=False)\n session.commit()\n log_pl.reflect(\"Task logs deleted\", payload={\"task_id\": task_id})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to delete task logs\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to delete logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_task:Function]\n\n # [DEF:delete_logs_for_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for multiple tasks.\n # @PRE: task_ids is a list of task IDs.\n # @POST: All logs for the tasks are deleted.\n # @PARAM: task_ids (List[str]) - List of task IDs.\n # @SIDE_EFFECT: Deletes rows from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskLogPersistenceService.delete_logs_for_tasks\"):\n log_pl.reason(\"Deleting logs for multiple tasks\", payload={\"count\": len(task_ids)})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id.in_(task_ids)\n ).delete(synchronize_session=False)\n session.commit()\n log_pl.reflect(\"Logs deleted for multiple tasks\", payload={\"count\": len(task_ids)})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to delete logs for tasks\", error=str(e))\n logger.error(f\"Failed to delete logs for tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_tasks:Function]\n\n\n# [/DEF:TaskLogPersistenceService:Class]\n# [/DEF:TaskPersistenceModule:Module]\n" }, { "contract_id": "TaskPersistenceService", "contract_type": "Class", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 30, - "end_line": 343, + "start_line": 34, + "end_line": 358, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -45858,14 +46614,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:TaskPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, sqlalchemy\n# @PURPOSE: Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.\n# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskRecord]\n# @RELATION: [DEPENDS_ON] ->[Environment]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @INVARIANT: Persistence must handle potentially missing task fields natively.\n#\n# @TEST_CONTRACT: TaskPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"persist_task creates or updates a record\",\n# \"load_tasks retrieves valid Task instances\",\n# \"delete_tasks correctly removes records from the database\"\n# ]\n# }\n# @TEST_FIXTURE: valid_task_persistence -> {\"task_id\": \"123\", \"status\": \"PENDING\"}\n# @TEST_EDGE: persist_invalid_task_type -> raises Exception\n# @TEST_EDGE: load_corrupt_json_params -> handled gracefully\n# @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]\nclass TaskPersistenceService:\n # [DEF:_json_load_if_needed:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely load JSON strings from DB if necessary\n # @PRE: value is an arbitrary database value\n # @POST: Returns parsed JSON object, list, string, or primitive\n @staticmethod\n def _json_load_if_needed(value):\n with belief_scope(\"TaskPersistenceService._json_load_if_needed\"):\n if value is None:\n return None\n if isinstance(value, (dict, list)):\n return value\n if isinstance(value, str):\n stripped = value.strip()\n if stripped == \"\" or stripped.lower() == \"null\":\n return None\n try:\n return json.loads(stripped)\n except json.JSONDecodeError:\n return value\n return value\n\n # [/DEF:_json_load_if_needed:Function]\n\n # [DEF:_parse_datetime:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely parse a datetime string from the database\n # @PRE: value is an ISO string or datetime object\n # @POST: Returns datetime object or None\n @staticmethod\n def _parse_datetime(value):\n with belief_scope(\"TaskPersistenceService._parse_datetime\"):\n if value is None or isinstance(value, datetime):\n return value\n if isinstance(value, str):\n try:\n return datetime.fromisoformat(value)\n except ValueError:\n return None\n return None\n\n # [/DEF:_parse_datetime:Function]\n\n # [DEF:_resolve_environment_id:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints.\n # @PRE: Session is active\n # @POST: Returns existing environments.id or None when unresolved.\n # @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]\n # @RELATION: [DEPENDS_ON] ->[Environment]\n @staticmethod\n def _resolve_environment_id(\n session: Session, env_id: Optional[str]\n ) -> Optional[str]:\n with belief_scope(\"_resolve_environment_id\"):\n raw_value = str(env_id or \"\").strip()\n if not raw_value:\n return None\n\n # 1) Direct match by primary key.\n by_id = (\n session.query(Environment).filter(Environment.id == raw_value).first()\n )\n if by_id:\n return str(by_id.id)\n\n # 2) Exact match by name.\n by_name = (\n session.query(Environment).filter(Environment.name == raw_value).first()\n )\n if by_name:\n return str(by_name.id)\n\n # 3) Slug-like match (e.g. \"ss-dev\" -> \"SS DEV\").\n def normalize_token(value: str) -> str:\n lowered = str(value or \"\").strip().lower()\n return re.sub(r\"[^a-z0-9]+\", \"-\", lowered).strip(\"-\")\n\n target_token = normalize_token(raw_value)\n if not target_token:\n return None\n\n for env in session.query(Environment).all():\n if (\n normalize_token(env.id) == target_token\n or normalize_token(env.name) == target_token\n ):\n return str(env.id)\n\n return None\n\n # [/DEF:_resolve_environment_id:Function]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the persistence service.\n # @PRE: None.\n # @POST: Service is ready.\n def __init__(self):\n with belief_scope(\"TaskPersistenceService.__init__\"):\n # We use TasksSessionLocal from database.py\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:persist_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists or updates a single task in the database.\n # @PRE: isinstance(task, Task)\n # @POST: Task record created or updated in database.\n # @PARAM: task (Task) - The task object to persist.\n # @SIDE_EFFECT: Writes to task_records table in tasks.db\n # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]\n # @RELATION: [CALLS] ->[_resolve_environment_id]\n def persist_task(self, task: Task) -> None:\n with belief_scope(\"TaskPersistenceService.persist_task\", f\"task_id={task.id}\"):\n session: Session = TasksSessionLocal()\n try:\n record = (\n session.query(TaskRecord).filter(TaskRecord.id == task.id).first()\n )\n if not record:\n record = TaskRecord(id=task.id)\n session.add(record)\n\n record.type = task.plugin_id\n record.status = task.status.value\n raw_env_id = task.params.get(\"environment_id\") or task.params.get(\n \"source_env_id\"\n )\n record.environment_id = self._resolve_environment_id(\n session, raw_env_id\n )\n record.started_at = task.started_at\n record.finished_at = task.finished_at\n\n # Ensure params and result are JSON serializable\n def json_serializable(obj):\n with belief_scope(\"TaskPersistenceService.json_serializable\"):\n if isinstance(obj, dict):\n return {k: json_serializable(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [json_serializable(v) for v in obj]\n elif isinstance(obj, datetime):\n return obj.isoformat()\n return obj\n\n record.params = json_serializable(task.params)\n record.result = json_serializable(task.result)\n\n # Store logs as JSON, converting datetime to string\n record.logs = []\n for log in task.logs:\n log_dict = log.dict()\n if isinstance(log_dict.get(\"timestamp\"), datetime):\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n # Also clean up any datetimes in context\n if log_dict.get(\"context\"):\n log_dict[\"context\"] = json_serializable(log_dict[\"context\"])\n record.logs.append(log_dict)\n\n # Extract error if failed\n if task.status == TaskStatus.FAILED:\n for log in reversed(task.logs):\n if log.level == \"ERROR\":\n record.error = log.message\n break\n\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to persist task {task.id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:persist_task:Function]\n\n # [DEF:persist_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists multiple tasks.\n # @PRE: isinstance(tasks, list)\n # @POST: All tasks in list are persisted.\n # @PARAM: tasks (List[Task]) - The list of tasks to persist.\n # @RELATION: [CALLS] ->[persist_task]\n def persist_tasks(self, tasks: List[Task]) -> None:\n with belief_scope(\"TaskPersistenceService.persist_tasks\"):\n for task in tasks:\n self.persist_task(task)\n\n # [/DEF:persist_tasks:Function]\n\n # [DEF:load_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Loads tasks from the database.\n # @PRE: limit is an integer.\n # @POST: Returns list of Task objects.\n # @PARAM: limit (int) - Max tasks to load.\n # @PARAM: status (Optional[TaskStatus]) - Filter by status.\n # @RETURN: List[Task] - The loaded tasks.\n # @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]\n # @RELATION: [CALLS] ->[_json_load_if_needed]\n # @RELATION: [CALLS] ->[_parse_datetime]\n def load_tasks(\n self, limit: int = 100, status: Optional[TaskStatus] = None\n ) -> List[Task]:\n with belief_scope(\"TaskPersistenceService.load_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskRecord)\n if status:\n query = query.filter(TaskRecord.status == status.value)\n\n records = (\n query.order_by(TaskRecord.created_at.desc()).limit(limit).all()\n )\n\n loaded_tasks = []\n for record in records:\n try:\n logs = []\n logs_payload = self._json_load_if_needed(record.logs)\n if isinstance(logs_payload, list):\n for log_data in logs_payload:\n if not isinstance(log_data, dict):\n continue\n log_data = dict(log_data)\n log_data[\"timestamp\"] = (\n self._parse_datetime(log_data.get(\"timestamp\"))\n or datetime.utcnow()\n )\n logs.append(LogEntry(**log_data))\n\n started_at = self._parse_datetime(record.started_at)\n finished_at = self._parse_datetime(record.finished_at)\n params = self._json_load_if_needed(record.params)\n result = self._json_load_if_needed(record.result)\n\n task = Task(\n id=record.id,\n plugin_id=record.type,\n status=TaskStatus(record.status),\n started_at=started_at,\n finished_at=finished_at,\n params=params if isinstance(params, dict) else {},\n result=result,\n logs=logs,\n )\n loaded_tasks.append(task)\n except Exception as e:\n logger.error(f\"Failed to reconstruct task {record.id}: {e}\")\n\n return loaded_tasks\n finally:\n session.close()\n\n # [/DEF:load_tasks:Function]\n\n # [DEF:delete_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Deletes specific tasks from the database.\n # @PRE: task_ids is a list of strings.\n # @POST: Specified task records deleted from database.\n # @PARAM: task_ids (List[str]) - List of task IDs to delete.\n # @SIDE_EFFECT: Deletes rows from task_records table.\n # @RELATION: [DEPENDS_ON] ->[TaskRecord]\n def delete_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskPersistenceService.delete_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(\n synchronize_session=False\n )\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_tasks:Function]\n\n\n# [/DEF:TaskPersistenceService:Class]\n" + "body": "# [DEF:TaskPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, sqlalchemy\n# @PURPOSE: Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.\n# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskRecord]\n# @RELATION: [DEPENDS_ON] ->[Environment]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[TaskGraph]\n# @INVARIANT: Persistence must handle potentially missing task fields natively.\n#\n# @TEST_CONTRACT: TaskPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"persist_task creates or updates a record\",\n# \"load_tasks retrieves valid Task instances\",\n# \"delete_tasks correctly removes records from the database\"\n# ]\n# }\n# @TEST_FIXTURE: valid_task_persistence -> {\"task_id\": \"123\", \"status\": \"PENDING\"}\n# @TEST_EDGE: persist_invalid_task_type -> raises Exception\n# @TEST_EDGE: load_corrupt_json_params -> handled gracefully\n# @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]\nclass TaskPersistenceService:\n # [DEF:_json_load_if_needed:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely load JSON strings from DB if necessary\n # @PRE: value is an arbitrary database value\n # @POST: Returns parsed JSON object, list, string, or primitive\n @staticmethod\n def _json_load_if_needed(value):\n with belief_scope(\"TaskPersistenceService._json_load_if_needed\"):\n if value is None:\n return None\n if isinstance(value, (dict, list)):\n return value\n if isinstance(value, str):\n stripped = value.strip()\n if stripped == \"\" or stripped.lower() == \"null\":\n return None\n try:\n return json.loads(stripped)\n except json.JSONDecodeError:\n return value\n return value\n\n # [/DEF:_json_load_if_needed:Function]\n\n # [DEF:_parse_datetime:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Safely parse a datetime string from the database\n # @PRE: value is an ISO string or datetime object\n # @POST: Returns datetime object or None\n @staticmethod\n def _parse_datetime(value):\n with belief_scope(\"TaskPersistenceService._parse_datetime\"):\n if value is None or isinstance(value, datetime):\n return value\n if isinstance(value, str):\n try:\n return datetime.fromisoformat(value)\n except ValueError:\n return None\n return None\n\n # [/DEF:_parse_datetime:Function]\n\n # [DEF:_resolve_environment_id:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints.\n # @PRE: Session is active\n # @POST: Returns existing environments.id or None when unresolved.\n # @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]\n # @RELATION: [DEPENDS_ON] ->[Environment]\n @staticmethod\n def _resolve_environment_id(\n session: Session, env_id: Optional[str]\n ) -> Optional[str]:\n with belief_scope(\"_resolve_environment_id\"):\n raw_value = str(env_id or \"\").strip()\n if not raw_value:\n return None\n\n # 1) Direct match by primary key.\n by_id = (\n session.query(Environment).filter(Environment.id == raw_value).first()\n )\n if by_id:\n return str(by_id.id)\n\n # 2) Exact match by name.\n by_name = (\n session.query(Environment).filter(Environment.name == raw_value).first()\n )\n if by_name:\n return str(by_name.id)\n\n # 3) Slug-like match (e.g. \"ss-dev\" -> \"SS DEV\").\n def normalize_token(value: str) -> str:\n lowered = str(value or \"\").strip().lower()\n return re.sub(r\"[^a-z0-9]+\", \"-\", lowered).strip(\"-\")\n\n target_token = normalize_token(raw_value)\n if not target_token:\n return None\n\n for env in session.query(Environment).all():\n if (\n normalize_token(env.id) == target_token\n or normalize_token(env.name) == target_token\n ):\n return str(env.id)\n\n return None\n\n # [/DEF:_resolve_environment_id:Function]\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the persistence service.\n # @PRE: None.\n # @POST: Service is ready.\n def __init__(self):\n with belief_scope(\"TaskPersistenceService.__init__\"):\n # We use TasksSessionLocal from database.py\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:persist_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists or updates a single task in the database.\n # @PRE: isinstance(task, Task)\n # @POST: Task record created or updated in database.\n # @PARAM: task (Task) - The task object to persist.\n # @SIDE_EFFECT: Writes to task_records table in tasks.db\n # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]\n # @RELATION: [CALLS] ->[_resolve_environment_id]\n def persist_task(self, task: Task) -> None:\n with belief_scope(\"TaskPersistenceService.persist_task\", f\"task_id={task.id}\"):\n log.reason(\"Persisting task to database\", payload={\"task_id\": task.id})\n session: Session = TasksSessionLocal()\n try:\n record = (\n session.query(TaskRecord).filter(TaskRecord.id == task.id).first()\n )\n if not record:\n record = TaskRecord(id=task.id)\n session.add(record)\n\n record.type = task.plugin_id\n record.status = task.status.value\n raw_env_id = task.params.get(\"environment_id\") or task.params.get(\n \"source_env_id\"\n )\n record.environment_id = self._resolve_environment_id(\n session, raw_env_id\n )\n record.started_at = task.started_at\n record.finished_at = task.finished_at\n\n # Ensure params and result are JSON serializable\n def json_serializable(obj):\n with belief_scope(\"TaskPersistenceService.json_serializable\"):\n if isinstance(obj, dict):\n return {k: json_serializable(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [json_serializable(v) for v in obj]\n elif isinstance(obj, datetime):\n return obj.isoformat()\n return obj\n\n record.params = json_serializable(task.params)\n record.result = json_serializable(task.result)\n\n # Store logs as JSON, converting datetime to string\n record.logs = []\n for log_entry in task.logs:\n log_dict = log_entry.dict()\n if isinstance(log_dict.get(\"timestamp\"), datetime):\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n # Also clean up any datetimes in context\n if log_dict.get(\"context\"):\n log_dict[\"context\"] = json_serializable(log_dict[\"context\"])\n record.logs.append(log_dict)\n\n # Extract error if failed\n if task.status == TaskStatus.FAILED:\n for log_entry in reversed(task.logs):\n if log_entry.level == \"ERROR\":\n record.error = log_entry.message\n break\n\n session.commit()\n log.reflect(\"Task persisted successfully\", payload={\"task_id\": task.id})\n except Exception as e:\n session.rollback()\n log.explore(\"Failed to persist task\", error=str(e), payload={\"task_id\": task.id})\n logger.error(f\"Failed to persist task {task.id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:persist_task:Function]\n\n # [DEF:persist_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists multiple tasks.\n # @PRE: isinstance(tasks, list)\n # @POST: All tasks in list are persisted.\n # @PARAM: tasks (List[Task]) - The list of tasks to persist.\n # @RELATION: [CALLS] ->[persist_task]\n def persist_tasks(self, tasks: List[Task]) -> None:\n with belief_scope(\"TaskPersistenceService.persist_tasks\"):\n log.reason(\"Persisting multiple tasks\", payload={\"count\": len(tasks)})\n for task in tasks:\n self.persist_task(task)\n log.reflect(\"All tasks persisted\", payload={\"count\": len(tasks)})\n\n # [/DEF:persist_tasks:Function]\n\n # [DEF:load_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Loads tasks from the database.\n # @PRE: limit is an integer.\n # @POST: Returns list of Task objects.\n # @PARAM: limit (int) - Max tasks to load.\n # @PARAM: status (Optional[TaskStatus]) - Filter by status.\n # @RETURN: List[Task] - The loaded tasks.\n # @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]\n # @RELATION: [CALLS] ->[_json_load_if_needed]\n # @RELATION: [CALLS] ->[_parse_datetime]\n def load_tasks(\n self, limit: int = 100, status: Optional[TaskStatus] = None\n ) -> List[Task]:\n with belief_scope(\"TaskPersistenceService.load_tasks\"):\n log.reason(\"Loading tasks from database\", payload={\"limit\": limit, \"status\": str(status) if status else None})\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskRecord)\n if status:\n query = query.filter(TaskRecord.status == status.value)\n\n records = (\n query.order_by(TaskRecord.created_at.desc()).limit(limit).all()\n )\n\n loaded_tasks = []\n for record in records:\n try:\n logs = []\n logs_payload = self._json_load_if_needed(record.logs)\n if isinstance(logs_payload, list):\n for log_data in logs_payload:\n if not isinstance(log_data, dict):\n continue\n log_data = dict(log_data)\n log_data[\"timestamp\"] = (\n self._parse_datetime(log_data.get(\"timestamp\"))\n or datetime.utcnow()\n )\n logs.append(LogEntry(**log_data))\n\n started_at = self._parse_datetime(record.started_at)\n finished_at = self._parse_datetime(record.finished_at)\n params = self._json_load_if_needed(record.params)\n result = self._json_load_if_needed(record.result)\n\n task = Task(\n id=record.id,\n plugin_id=record.type,\n status=TaskStatus(record.status),\n started_at=started_at,\n finished_at=finished_at,\n params=params if isinstance(params, dict) else {},\n result=result,\n logs=logs,\n )\n loaded_tasks.append(task)\n except Exception as e:\n log.explore(\"Failed to reconstruct task from record\", error=str(e), payload={\"record_id\": record.id})\n logger.error(f\"Failed to reconstruct task {record.id}: {e}\")\n\n log.reflect(\"Tasks loaded from database\", payload={\"count\": len(loaded_tasks)})\n return loaded_tasks\n finally:\n session.close()\n\n # [/DEF:load_tasks:Function]\n\n # [DEF:delete_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Deletes specific tasks from the database.\n # @PRE: task_ids is a list of strings.\n # @POST: Specified task records deleted from database.\n # @PARAM: task_ids (List[str]) - List of task IDs to delete.\n # @SIDE_EFFECT: Deletes rows from task_records table.\n # @RELATION: [DEPENDS_ON] ->[TaskRecord]\n def delete_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskPersistenceService.delete_tasks\"):\n log.reason(\"Deleting tasks from database\", payload={\"count\": len(task_ids)})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(\n synchronize_session=False\n )\n session.commit()\n log.reflect(\"Tasks deleted from database\", payload={\"count\": len(task_ids)})\n except Exception as e:\n session.rollback()\n log.explore(\"Failed to delete tasks\", error=str(e))\n logger.error(f\"Failed to delete tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_tasks:Function]\n\n\n# [/DEF:TaskPersistenceService:Class]\n" }, { "contract_id": "_parse_datetime", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 83, - "end_line": 100, + "start_line": 87, + "end_line": 104, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -45902,8 +46658,8 @@ "contract_id": "_resolve_environment_id", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 102, - "end_line": 150, + "start_line": 106, + "end_line": 154, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -45974,8 +46730,8 @@ "contract_id": "persist_task", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 164, - "end_line": 234, + "start_line": 168, + "end_line": 241, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46057,14 +46813,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:persist_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists or updates a single task in the database.\n # @PRE: isinstance(task, Task)\n # @POST: Task record created or updated in database.\n # @PARAM: task (Task) - The task object to persist.\n # @SIDE_EFFECT: Writes to task_records table in tasks.db\n # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]\n # @RELATION: [CALLS] ->[_resolve_environment_id]\n def persist_task(self, task: Task) -> None:\n with belief_scope(\"TaskPersistenceService.persist_task\", f\"task_id={task.id}\"):\n session: Session = TasksSessionLocal()\n try:\n record = (\n session.query(TaskRecord).filter(TaskRecord.id == task.id).first()\n )\n if not record:\n record = TaskRecord(id=task.id)\n session.add(record)\n\n record.type = task.plugin_id\n record.status = task.status.value\n raw_env_id = task.params.get(\"environment_id\") or task.params.get(\n \"source_env_id\"\n )\n record.environment_id = self._resolve_environment_id(\n session, raw_env_id\n )\n record.started_at = task.started_at\n record.finished_at = task.finished_at\n\n # Ensure params and result are JSON serializable\n def json_serializable(obj):\n with belief_scope(\"TaskPersistenceService.json_serializable\"):\n if isinstance(obj, dict):\n return {k: json_serializable(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [json_serializable(v) for v in obj]\n elif isinstance(obj, datetime):\n return obj.isoformat()\n return obj\n\n record.params = json_serializable(task.params)\n record.result = json_serializable(task.result)\n\n # Store logs as JSON, converting datetime to string\n record.logs = []\n for log in task.logs:\n log_dict = log.dict()\n if isinstance(log_dict.get(\"timestamp\"), datetime):\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n # Also clean up any datetimes in context\n if log_dict.get(\"context\"):\n log_dict[\"context\"] = json_serializable(log_dict[\"context\"])\n record.logs.append(log_dict)\n\n # Extract error if failed\n if task.status == TaskStatus.FAILED:\n for log in reversed(task.logs):\n if log.level == \"ERROR\":\n record.error = log.message\n break\n\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to persist task {task.id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:persist_task:Function]\n" + "body": " # [DEF:persist_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists or updates a single task in the database.\n # @PRE: isinstance(task, Task)\n # @POST: Task record created or updated in database.\n # @PARAM: task (Task) - The task object to persist.\n # @SIDE_EFFECT: Writes to task_records table in tasks.db\n # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]\n # @RELATION: [CALLS] ->[_resolve_environment_id]\n def persist_task(self, task: Task) -> None:\n with belief_scope(\"TaskPersistenceService.persist_task\", f\"task_id={task.id}\"):\n log.reason(\"Persisting task to database\", payload={\"task_id\": task.id})\n session: Session = TasksSessionLocal()\n try:\n record = (\n session.query(TaskRecord).filter(TaskRecord.id == task.id).first()\n )\n if not record:\n record = TaskRecord(id=task.id)\n session.add(record)\n\n record.type = task.plugin_id\n record.status = task.status.value\n raw_env_id = task.params.get(\"environment_id\") or task.params.get(\n \"source_env_id\"\n )\n record.environment_id = self._resolve_environment_id(\n session, raw_env_id\n )\n record.started_at = task.started_at\n record.finished_at = task.finished_at\n\n # Ensure params and result are JSON serializable\n def json_serializable(obj):\n with belief_scope(\"TaskPersistenceService.json_serializable\"):\n if isinstance(obj, dict):\n return {k: json_serializable(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [json_serializable(v) for v in obj]\n elif isinstance(obj, datetime):\n return obj.isoformat()\n return obj\n\n record.params = json_serializable(task.params)\n record.result = json_serializable(task.result)\n\n # Store logs as JSON, converting datetime to string\n record.logs = []\n for log_entry in task.logs:\n log_dict = log_entry.dict()\n if isinstance(log_dict.get(\"timestamp\"), datetime):\n log_dict[\"timestamp\"] = log_dict[\"timestamp\"].isoformat()\n # Also clean up any datetimes in context\n if log_dict.get(\"context\"):\n log_dict[\"context\"] = json_serializable(log_dict[\"context\"])\n record.logs.append(log_dict)\n\n # Extract error if failed\n if task.status == TaskStatus.FAILED:\n for log_entry in reversed(task.logs):\n if log_entry.level == \"ERROR\":\n record.error = log_entry.message\n break\n\n session.commit()\n log.reflect(\"Task persisted successfully\", payload={\"task_id\": task.id})\n except Exception as e:\n session.rollback()\n log.explore(\"Failed to persist task\", error=str(e), payload={\"task_id\": task.id})\n logger.error(f\"Failed to persist task {task.id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:persist_task:Function]\n" }, { "contract_id": "persist_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 236, - "end_line": 248, + "start_line": 243, + "end_line": 257, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46126,14 +46882,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:persist_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists multiple tasks.\n # @PRE: isinstance(tasks, list)\n # @POST: All tasks in list are persisted.\n # @PARAM: tasks (List[Task]) - The list of tasks to persist.\n # @RELATION: [CALLS] ->[persist_task]\n def persist_tasks(self, tasks: List[Task]) -> None:\n with belief_scope(\"TaskPersistenceService.persist_tasks\"):\n for task in tasks:\n self.persist_task(task)\n\n # [/DEF:persist_tasks:Function]\n" + "body": " # [DEF:persist_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persists multiple tasks.\n # @PRE: isinstance(tasks, list)\n # @POST: All tasks in list are persisted.\n # @PARAM: tasks (List[Task]) - The list of tasks to persist.\n # @RELATION: [CALLS] ->[persist_task]\n def persist_tasks(self, tasks: List[Task]) -> None:\n with belief_scope(\"TaskPersistenceService.persist_tasks\"):\n log.reason(\"Persisting multiple tasks\", payload={\"count\": len(tasks)})\n for task in tasks:\n self.persist_task(task)\n log.reflect(\"All tasks persisted\", payload={\"count\": len(tasks)})\n\n # [/DEF:persist_tasks:Function]\n" }, { "contract_id": "load_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 250, - "end_line": 314, + "start_line": 259, + "end_line": 326, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46235,14 +46991,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:load_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Loads tasks from the database.\n # @PRE: limit is an integer.\n # @POST: Returns list of Task objects.\n # @PARAM: limit (int) - Max tasks to load.\n # @PARAM: status (Optional[TaskStatus]) - Filter by status.\n # @RETURN: List[Task] - The loaded tasks.\n # @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]\n # @RELATION: [CALLS] ->[_json_load_if_needed]\n # @RELATION: [CALLS] ->[_parse_datetime]\n def load_tasks(\n self, limit: int = 100, status: Optional[TaskStatus] = None\n ) -> List[Task]:\n with belief_scope(\"TaskPersistenceService.load_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskRecord)\n if status:\n query = query.filter(TaskRecord.status == status.value)\n\n records = (\n query.order_by(TaskRecord.created_at.desc()).limit(limit).all()\n )\n\n loaded_tasks = []\n for record in records:\n try:\n logs = []\n logs_payload = self._json_load_if_needed(record.logs)\n if isinstance(logs_payload, list):\n for log_data in logs_payload:\n if not isinstance(log_data, dict):\n continue\n log_data = dict(log_data)\n log_data[\"timestamp\"] = (\n self._parse_datetime(log_data.get(\"timestamp\"))\n or datetime.utcnow()\n )\n logs.append(LogEntry(**log_data))\n\n started_at = self._parse_datetime(record.started_at)\n finished_at = self._parse_datetime(record.finished_at)\n params = self._json_load_if_needed(record.params)\n result = self._json_load_if_needed(record.result)\n\n task = Task(\n id=record.id,\n plugin_id=record.type,\n status=TaskStatus(record.status),\n started_at=started_at,\n finished_at=finished_at,\n params=params if isinstance(params, dict) else {},\n result=result,\n logs=logs,\n )\n loaded_tasks.append(task)\n except Exception as e:\n logger.error(f\"Failed to reconstruct task {record.id}: {e}\")\n\n return loaded_tasks\n finally:\n session.close()\n\n # [/DEF:load_tasks:Function]\n" + "body": " # [DEF:load_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Loads tasks from the database.\n # @PRE: limit is an integer.\n # @POST: Returns list of Task objects.\n # @PARAM: limit (int) - Max tasks to load.\n # @PARAM: status (Optional[TaskStatus]) - Filter by status.\n # @RETURN: List[Task] - The loaded tasks.\n # @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]\n # @RELATION: [CALLS] ->[_json_load_if_needed]\n # @RELATION: [CALLS] ->[_parse_datetime]\n def load_tasks(\n self, limit: int = 100, status: Optional[TaskStatus] = None\n ) -> List[Task]:\n with belief_scope(\"TaskPersistenceService.load_tasks\"):\n log.reason(\"Loading tasks from database\", payload={\"limit\": limit, \"status\": str(status) if status else None})\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskRecord)\n if status:\n query = query.filter(TaskRecord.status == status.value)\n\n records = (\n query.order_by(TaskRecord.created_at.desc()).limit(limit).all()\n )\n\n loaded_tasks = []\n for record in records:\n try:\n logs = []\n logs_payload = self._json_load_if_needed(record.logs)\n if isinstance(logs_payload, list):\n for log_data in logs_payload:\n if not isinstance(log_data, dict):\n continue\n log_data = dict(log_data)\n log_data[\"timestamp\"] = (\n self._parse_datetime(log_data.get(\"timestamp\"))\n or datetime.utcnow()\n )\n logs.append(LogEntry(**log_data))\n\n started_at = self._parse_datetime(record.started_at)\n finished_at = self._parse_datetime(record.finished_at)\n params = self._json_load_if_needed(record.params)\n result = self._json_load_if_needed(record.result)\n\n task = Task(\n id=record.id,\n plugin_id=record.type,\n status=TaskStatus(record.status),\n started_at=started_at,\n finished_at=finished_at,\n params=params if isinstance(params, dict) else {},\n result=result,\n logs=logs,\n )\n loaded_tasks.append(task)\n except Exception as e:\n log.explore(\"Failed to reconstruct task from record\", error=str(e), payload={\"record_id\": record.id})\n logger.error(f\"Failed to reconstruct task {record.id}: {e}\")\n\n log.reflect(\"Tasks loaded from database\", payload={\"count\": len(loaded_tasks)})\n return loaded_tasks\n finally:\n session.close()\n\n # [/DEF:load_tasks:Function]\n" }, { "contract_id": "delete_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 316, - "end_line": 340, + "start_line": 328, + "end_line": 355, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46314,14 +47070,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:delete_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Deletes specific tasks from the database.\n # @PRE: task_ids is a list of strings.\n # @POST: Specified task records deleted from database.\n # @PARAM: task_ids (List[str]) - List of task IDs to delete.\n # @SIDE_EFFECT: Deletes rows from task_records table.\n # @RELATION: [DEPENDS_ON] ->[TaskRecord]\n def delete_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskPersistenceService.delete_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(\n synchronize_session=False\n )\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_tasks:Function]\n" + "body": " # [DEF:delete_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Deletes specific tasks from the database.\n # @PRE: task_ids is a list of strings.\n # @POST: Specified task records deleted from database.\n # @PARAM: task_ids (List[str]) - List of task IDs to delete.\n # @SIDE_EFFECT: Deletes rows from task_records table.\n # @RELATION: [DEPENDS_ON] ->[TaskRecord]\n def delete_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskPersistenceService.delete_tasks\"):\n log.reason(\"Deleting tasks from database\", payload={\"count\": len(task_ids)})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(\n synchronize_session=False\n )\n session.commit()\n log.reflect(\"Tasks deleted from database\", payload={\"count\": len(task_ids)})\n except Exception as e:\n session.rollback()\n log.explore(\"Failed to delete tasks\", error=str(e))\n logger.error(f\"Failed to delete tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_tasks:Function]\n" }, { "contract_id": "TaskLogPersistenceService", "contract_type": "Class", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 346, - "end_line": 623, + "start_line": 361, + "end_line": 654, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -46479,14 +47235,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:TaskLogPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, log, sqlalchemy\n# @PURPOSE: Provides methods to store, query, summarize, and delete task log rows in the task_logs table.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.\n# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]\n# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @INVARIANT: Log entries are batch-inserted for performance.\n#\n# @TEST_CONTRACT: TaskLogPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"add_logs efficiently saves logs to the database\",\n# \"get_logs retrieves properly filtered LogEntry objects\"\n# ]\n# }\n# @TEST_FIXTURE: valid_log_batch -> {\"task_id\": \"123\", \"logs\": [{\"level\": \"INFO\", \"message\": \"msg\"}]}\n# @TEST_EDGE: empty_log_list -> no-op behavior\n# @TEST_EDGE: add_logs_db_error -> rollback and log error\n# @TEST_INVARIANT: accurate_log_aggregation -> verifies: [valid_log_batch]\nclass TaskLogPersistenceService:\n \"\"\"\n Service for persisting and querying task logs.\n Supports batch inserts, filtering, and statistics.\n \"\"\"\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the TaskLogPersistenceService\n # @PRE: config is provided or defaults are used\n # @POST: Service is ready for log persistence\n def __init__(self, config=None):\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:add_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Batch insert log entries for a task.\n # @PRE: logs is a list of LogEntry objects.\n # @POST: All logs inserted into task_logs table.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: logs (List[LogEntry]) - Log entries to insert.\n # @SIDE_EFFECT: Writes to task_logs table.\n # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:\n if not logs:\n return\n with belief_scope(\"TaskLogPersistenceService.add_logs\", f\"task_id={task_id}\"):\n session: Session = TasksSessionLocal()\n try:\n for log in logs:\n record = TaskLogRecord(\n task_id=task_id,\n timestamp=log.timestamp,\n level=log.level,\n source=log.source or \"system\",\n message=log.message,\n metadata_json=json.dumps(log.metadata)\n if log.metadata\n else None,\n )\n session.add(record)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to add logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:add_logs:Function]\n\n # [DEF:get_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Query logs for a task with filtering and pagination.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of TaskLog objects matching filters.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: log_filter (LogFilter) - Filter parameters.\n # @RETURN: List[TaskLog] - Filtered log entries.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogFilter]\n # @RELATION: [DEPENDS_ON] ->[TaskLog]\n def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:\n with belief_scope(\"TaskLogPersistenceService.get_logs\", f\"task_id={task_id}\"):\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n )\n\n # Apply filters\n if log_filter.level:\n query = query.filter(\n TaskLogRecord.level == log_filter.level.upper()\n )\n if log_filter.source:\n query = query.filter(TaskLogRecord.source == log_filter.source)\n if log_filter.search:\n search_pattern = f\"%{log_filter.search}%\"\n query = query.filter(TaskLogRecord.message.ilike(search_pattern))\n\n # Order by timestamp ascending (oldest first)\n query = query.order_by(TaskLogRecord.timestamp.asc())\n\n # Apply pagination\n records = query.offset(log_filter.offset).limit(log_filter.limit).all()\n\n logs = []\n for record in records:\n metadata = None\n if record.metadata_json:\n try:\n metadata = json.loads(record.metadata_json)\n except json.JSONDecodeError:\n metadata = None\n\n logs.append(\n TaskLog(\n id=record.id,\n task_id=record.task_id,\n timestamp=record.timestamp,\n level=record.level,\n source=record.source,\n message=record.message,\n metadata=metadata,\n )\n )\n\n return logs\n finally:\n session.close()\n\n # [/DEF:get_logs:Function]\n\n # [DEF:get_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogStats]\n def get_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\n \"TaskLogPersistenceService.get_log_stats\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n # Get total count\n total_count = (\n session.query(TaskLogRecord)\n .filter(TaskLogRecord.task_id == task_id)\n .count()\n )\n\n # Get counts by level\n from sqlalchemy import func\n\n level_counts = (\n session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.level)\n .all()\n )\n\n by_level = {level: count for level, count in level_counts}\n\n # Get counts by source\n source_counts = (\n session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.source)\n .all()\n )\n\n by_source = {source: count for source, count in source_counts}\n\n return LogStats(\n total_count=total_count, by_level=by_level, by_source=by_source\n )\n finally:\n session.close()\n\n # [/DEF:get_log_stats:Function]\n\n # [DEF:get_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def get_sources(self, task_id: str) -> List[str]:\n with belief_scope(\n \"TaskLogPersistenceService.get_sources\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n from sqlalchemy import distinct\n\n sources = (\n session.query(distinct(TaskLogRecord.source))\n .filter(TaskLogRecord.task_id == task_id)\n .all()\n )\n return [s[0] for s in sources]\n finally:\n session.close()\n\n # [/DEF:get_sources:Function]\n\n # [DEF:delete_logs_for_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for a specific task.\n # @PRE: task_id is a valid task ID.\n # @POST: All logs for the task are deleted.\n # @PARAM: task_id (str) - The task ID.\n # @SIDE_EFFECT: Deletes from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_task(self, task_id: str) -> None:\n with belief_scope(\n \"TaskLogPersistenceService.delete_logs_for_task\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n ).delete(synchronize_session=False)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_task:Function]\n\n # [DEF:delete_logs_for_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for multiple tasks.\n # @PRE: task_ids is a list of task IDs.\n # @POST: All logs for the tasks are deleted.\n # @PARAM: task_ids (List[str]) - List of task IDs.\n # @SIDE_EFFECT: Deletes rows from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskLogPersistenceService.delete_logs_for_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id.in_(task_ids)\n ).delete(synchronize_session=False)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete logs for tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_tasks:Function]\n\n\n# [/DEF:TaskLogPersistenceService:Class]\n" + "body": "# [DEF:TaskLogPersistenceService:Class]\n# @COMPLEXITY: 5\n# @SEMANTICS: persistence, service, database, log, sqlalchemy\n# @PURPOSE: Provides methods to store, query, summarize, and delete task log rows in the task_logs table.\n# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.\n# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.\n# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.\n# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]\n# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]\n# @RELATION: [DEPENDS_ON] ->[TaskManager]\n# @RELATION: [DEPENDS_ON] ->[EventBus]\n# @INVARIANT: Log entries are batch-inserted for performance.\n#\n# @TEST_CONTRACT: TaskLogPersistenceContract ->\n# {\n# required_fields: {},\n# invariants: [\n# \"add_logs efficiently saves logs to the database\",\n# \"get_logs retrieves properly filtered LogEntry objects\"\n# ]\n# }\n# @TEST_FIXTURE: valid_log_batch -> {\"task_id\": \"123\", \"logs\": [{\"level\": \"INFO\", \"message\": \"msg\"}]}\n# @TEST_EDGE: empty_log_list -> no-op behavior\n# @TEST_EDGE: add_logs_db_error -> rollback and log error\n# @TEST_INVARIANT: accurate_log_aggregation -> verifies: [valid_log_batch]\nclass TaskLogPersistenceService:\n \"\"\"\n Service for persisting and querying task logs.\n Supports batch inserts, filtering, and statistics.\n \"\"\"\n\n # [DEF:__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initializes the TaskLogPersistenceService\n # @PRE: config is provided or defaults are used\n # @POST: Service is ready for log persistence\n def __init__(self, config=None):\n pass\n\n # [/DEF:__init__:Function]\n\n # [DEF:add_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Batch insert log entries for a task.\n # @PRE: logs is a list of LogEntry objects.\n # @POST: All logs inserted into task_logs table.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: logs (List[LogEntry]) - Log entries to insert.\n # @SIDE_EFFECT: Writes to task_logs table.\n # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:\n if not logs:\n return\n with belief_scope(\"TaskLogPersistenceService.add_logs\", f\"task_id={task_id}\"):\n log_pl.reason(\"Batch inserting log entries\", payload={\"task_id\": task_id, \"count\": len(logs)})\n session: Session = TasksSessionLocal()\n try:\n for log_entry in logs:\n record = TaskLogRecord(\n task_id=task_id,\n timestamp=log_entry.timestamp,\n level=log_entry.level,\n source=log_entry.source or \"system\",\n message=log_entry.message,\n metadata_json=json.dumps(log_entry.metadata)\n if log_entry.metadata\n else None,\n )\n session.add(record)\n session.commit()\n log_pl.reflect(\"Log entries persisted\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to persist log entries\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to add logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:add_logs:Function]\n\n # [DEF:get_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Query logs for a task with filtering and pagination.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of TaskLog objects matching filters.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: log_filter (LogFilter) - Filter parameters.\n # @RETURN: List[TaskLog] - Filtered log entries.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogFilter]\n # @RELATION: [DEPENDS_ON] ->[TaskLog]\n def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:\n with belief_scope(\"TaskLogPersistenceService.get_logs\", f\"task_id={task_id}\"):\n log_pl.reason(\"Querying task logs\", payload={\"task_id\": task_id, \"filter_level\": log_filter.level})\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n )\n\n # Apply filters\n if log_filter.level:\n query = query.filter(\n TaskLogRecord.level == log_filter.level.upper()\n )\n if log_filter.source:\n query = query.filter(TaskLogRecord.source == log_filter.source)\n if log_filter.search:\n search_pattern = f\"%{log_filter.search}%\"\n query = query.filter(TaskLogRecord.message.ilike(search_pattern))\n\n # Order by timestamp ascending (oldest first)\n query = query.order_by(TaskLogRecord.timestamp.asc())\n\n # Apply pagination\n records = query.offset(log_filter.offset).limit(log_filter.limit).all()\n\n logs = []\n for record in records:\n metadata = None\n if record.metadata_json:\n try:\n metadata = json.loads(record.metadata_json)\n except json.JSONDecodeError:\n metadata = None\n\n logs.append(\n TaskLog(\n id=record.id,\n task_id=record.task_id,\n timestamp=record.timestamp,\n level=record.level,\n source=record.source,\n message=record.message,\n metadata=metadata,\n )\n )\n\n log_pl.reflect(\"Task logs retrieved\", payload={\"task_id\": task_id, \"count\": len(logs)})\n return logs\n finally:\n session.close()\n\n # [/DEF:get_logs:Function]\n\n # [DEF:get_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogStats]\n def get_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\n \"TaskLogPersistenceService.get_log_stats\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Aggregating log statistics\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n # Get total count\n total_count = (\n session.query(TaskLogRecord)\n .filter(TaskLogRecord.task_id == task_id)\n .count()\n )\n\n # Get counts by level\n from sqlalchemy import func\n\n level_counts = (\n session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.level)\n .all()\n )\n\n by_level = {level: count for level, count in level_counts}\n\n # Get counts by source\n source_counts = (\n session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.source)\n .all()\n )\n\n by_source = {source: count for source, count in source_counts}\n\n log_pl.reflect(\"Log statistics computed\", payload={\"task_id\": task_id, \"total\": total_count})\n return LogStats(\n total_count=total_count, by_level=by_level, by_source=by_source\n )\n finally:\n session.close()\n\n # [/DEF:get_log_stats:Function]\n\n # [DEF:get_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def get_sources(self, task_id: str) -> List[str]:\n with belief_scope(\n \"TaskLogPersistenceService.get_sources\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Querying distinct log sources\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n from sqlalchemy import distinct\n\n sources = (\n session.query(distinct(TaskLogRecord.source))\n .filter(TaskLogRecord.task_id == task_id)\n .all()\n )\n result = [s[0] for s in sources]\n log_pl.reflect(\"Log sources retrieved\", payload={\"task_id\": task_id, \"sources\": result})\n return result\n finally:\n session.close()\n\n # [/DEF:get_sources:Function]\n\n # [DEF:delete_logs_for_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for a specific task.\n # @PRE: task_id is a valid task ID.\n # @POST: All logs for the task are deleted.\n # @PARAM: task_id (str) - The task ID.\n # @SIDE_EFFECT: Deletes from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_task(self, task_id: str) -> None:\n with belief_scope(\n \"TaskLogPersistenceService.delete_logs_for_task\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Deleting logs for task\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n ).delete(synchronize_session=False)\n session.commit()\n log_pl.reflect(\"Task logs deleted\", payload={\"task_id\": task_id})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to delete task logs\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to delete logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_task:Function]\n\n # [DEF:delete_logs_for_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for multiple tasks.\n # @PRE: task_ids is a list of task IDs.\n # @POST: All logs for the tasks are deleted.\n # @PARAM: task_ids (List[str]) - List of task IDs.\n # @SIDE_EFFECT: Deletes rows from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskLogPersistenceService.delete_logs_for_tasks\"):\n log_pl.reason(\"Deleting logs for multiple tasks\", payload={\"count\": len(task_ids)})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id.in_(task_ids)\n ).delete(synchronize_session=False)\n session.commit()\n log_pl.reflect(\"Logs deleted for multiple tasks\", payload={\"count\": len(task_ids)})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to delete logs for tasks\", error=str(e))\n logger.error(f\"Failed to delete logs for tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_tasks:Function]\n\n\n# [/DEF:TaskLogPersistenceService:Class]\n" }, { "contract_id": "add_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 388, - "end_line": 423, + "start_line": 403, + "end_line": 441, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46568,14 +47324,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:add_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Batch insert log entries for a task.\n # @PRE: logs is a list of LogEntry objects.\n # @POST: All logs inserted into task_logs table.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: logs (List[LogEntry]) - Log entries to insert.\n # @SIDE_EFFECT: Writes to task_logs table.\n # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:\n if not logs:\n return\n with belief_scope(\"TaskLogPersistenceService.add_logs\", f\"task_id={task_id}\"):\n session: Session = TasksSessionLocal()\n try:\n for log in logs:\n record = TaskLogRecord(\n task_id=task_id,\n timestamp=log.timestamp,\n level=log.level,\n source=log.source or \"system\",\n message=log.message,\n metadata_json=json.dumps(log.metadata)\n if log.metadata\n else None,\n )\n session.add(record)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to add logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:add_logs:Function]\n" + "body": " # [DEF:add_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Batch insert log entries for a task.\n # @PRE: logs is a list of LogEntry objects.\n # @POST: All logs inserted into task_logs table.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: logs (List[LogEntry]) - Log entries to insert.\n # @SIDE_EFFECT: Writes to task_logs table.\n # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:\n if not logs:\n return\n with belief_scope(\"TaskLogPersistenceService.add_logs\", f\"task_id={task_id}\"):\n log_pl.reason(\"Batch inserting log entries\", payload={\"task_id\": task_id, \"count\": len(logs)})\n session: Session = TasksSessionLocal()\n try:\n for log_entry in logs:\n record = TaskLogRecord(\n task_id=task_id,\n timestamp=log_entry.timestamp,\n level=log_entry.level,\n source=log_entry.source or \"system\",\n message=log_entry.message,\n metadata_json=json.dumps(log_entry.metadata)\n if log_entry.metadata\n else None,\n )\n session.add(record)\n session.commit()\n log_pl.reflect(\"Log entries persisted\", payload={\"task_id\": task_id, \"count\": len(logs)})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to persist log entries\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to add logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:add_logs:Function]\n" }, { "contract_id": "get_logs", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 425, - "end_line": 487, + "start_line": 443, + "end_line": 507, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46700,14 +47456,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Query logs for a task with filtering and pagination.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of TaskLog objects matching filters.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: log_filter (LogFilter) - Filter parameters.\n # @RETURN: List[TaskLog] - Filtered log entries.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogFilter]\n # @RELATION: [DEPENDS_ON] ->[TaskLog]\n def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:\n with belief_scope(\"TaskLogPersistenceService.get_logs\", f\"task_id={task_id}\"):\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n )\n\n # Apply filters\n if log_filter.level:\n query = query.filter(\n TaskLogRecord.level == log_filter.level.upper()\n )\n if log_filter.source:\n query = query.filter(TaskLogRecord.source == log_filter.source)\n if log_filter.search:\n search_pattern = f\"%{log_filter.search}%\"\n query = query.filter(TaskLogRecord.message.ilike(search_pattern))\n\n # Order by timestamp ascending (oldest first)\n query = query.order_by(TaskLogRecord.timestamp.asc())\n\n # Apply pagination\n records = query.offset(log_filter.offset).limit(log_filter.limit).all()\n\n logs = []\n for record in records:\n metadata = None\n if record.metadata_json:\n try:\n metadata = json.loads(record.metadata_json)\n except json.JSONDecodeError:\n metadata = None\n\n logs.append(\n TaskLog(\n id=record.id,\n task_id=record.task_id,\n timestamp=record.timestamp,\n level=record.level,\n source=record.source,\n message=record.message,\n metadata=metadata,\n )\n )\n\n return logs\n finally:\n session.close()\n\n # [/DEF:get_logs:Function]\n" + "body": " # [DEF:get_logs:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Query logs for a task with filtering and pagination.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of TaskLog objects matching filters.\n # @PARAM: task_id (str) - The task ID.\n # @PARAM: log_filter (LogFilter) - Filter parameters.\n # @RETURN: List[TaskLog] - Filtered log entries.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogFilter]\n # @RELATION: [DEPENDS_ON] ->[TaskLog]\n def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:\n with belief_scope(\"TaskLogPersistenceService.get_logs\", f\"task_id={task_id}\"):\n log_pl.reason(\"Querying task logs\", payload={\"task_id\": task_id, \"filter_level\": log_filter.level})\n session: Session = TasksSessionLocal()\n try:\n query = session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n )\n\n # Apply filters\n if log_filter.level:\n query = query.filter(\n TaskLogRecord.level == log_filter.level.upper()\n )\n if log_filter.source:\n query = query.filter(TaskLogRecord.source == log_filter.source)\n if log_filter.search:\n search_pattern = f\"%{log_filter.search}%\"\n query = query.filter(TaskLogRecord.message.ilike(search_pattern))\n\n # Order by timestamp ascending (oldest first)\n query = query.order_by(TaskLogRecord.timestamp.asc())\n\n # Apply pagination\n records = query.offset(log_filter.offset).limit(log_filter.limit).all()\n\n logs = []\n for record in records:\n metadata = None\n if record.metadata_json:\n try:\n metadata = json.loads(record.metadata_json)\n except json.JSONDecodeError:\n metadata = None\n\n logs.append(\n TaskLog(\n id=record.id,\n task_id=record.task_id,\n timestamp=record.timestamp,\n level=record.level,\n source=record.source,\n message=record.message,\n metadata=metadata,\n )\n )\n\n log_pl.reflect(\"Task logs retrieved\", payload={\"task_id\": task_id, \"count\": len(logs)})\n return logs\n finally:\n session.close()\n\n # [/DEF:get_logs:Function]\n" }, { "contract_id": "get_log_stats", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 489, - "end_line": 540, + "start_line": 509, + "end_line": 562, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46809,14 +47565,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogStats]\n def get_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\n \"TaskLogPersistenceService.get_log_stats\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n # Get total count\n total_count = (\n session.query(TaskLogRecord)\n .filter(TaskLogRecord.task_id == task_id)\n .count()\n )\n\n # Get counts by level\n from sqlalchemy import func\n\n level_counts = (\n session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.level)\n .all()\n )\n\n by_level = {level: count for level, count in level_counts}\n\n # Get counts by source\n source_counts = (\n session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.source)\n .all()\n )\n\n by_source = {source: count for source, count in source_counts}\n\n return LogStats(\n total_count=total_count, by_level=by_level, by_source=by_source\n )\n finally:\n session.close()\n\n # [/DEF:get_log_stats:Function]\n" + "body": " # [DEF:get_log_stats:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get statistics about logs for a task.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns LogStats with counts by level and source.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: LogStats - Statistics about task logs.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n # @RELATION: [DEPENDS_ON] ->[LogStats]\n def get_log_stats(self, task_id: str) -> LogStats:\n with belief_scope(\n \"TaskLogPersistenceService.get_log_stats\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Aggregating log statistics\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n # Get total count\n total_count = (\n session.query(TaskLogRecord)\n .filter(TaskLogRecord.task_id == task_id)\n .count()\n )\n\n # Get counts by level\n from sqlalchemy import func\n\n level_counts = (\n session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.level)\n .all()\n )\n\n by_level = {level: count for level, count in level_counts}\n\n # Get counts by source\n source_counts = (\n session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))\n .filter(TaskLogRecord.task_id == task_id)\n .group_by(TaskLogRecord.source)\n .all()\n )\n\n by_source = {source: count for source, count in source_counts}\n\n log_pl.reflect(\"Log statistics computed\", payload={\"task_id\": task_id, \"total\": total_count})\n return LogStats(\n total_count=total_count, by_level=by_level, by_source=by_source\n )\n finally:\n session.close()\n\n # [/DEF:get_log_stats:Function]\n" }, { "contract_id": "get_sources", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 542, - "end_line": 568, + "start_line": 564, + "end_line": 593, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46895,14 +47651,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def get_sources(self, task_id: str) -> List[str]:\n with belief_scope(\n \"TaskLogPersistenceService.get_sources\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n from sqlalchemy import distinct\n\n sources = (\n session.query(distinct(TaskLogRecord.source))\n .filter(TaskLogRecord.task_id == task_id)\n .all()\n )\n return [s[0] for s in sources]\n finally:\n session.close()\n\n # [/DEF:get_sources:Function]\n" + "body": " # [DEF:get_sources:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get unique sources for a task's logs.\n # @PRE: task_id is a valid task ID.\n # @POST: Returns list of unique source strings.\n # @PARAM: task_id (str) - The task ID.\n # @RETURN: List[str] - Unique source names.\n # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def get_sources(self, task_id: str) -> List[str]:\n with belief_scope(\n \"TaskLogPersistenceService.get_sources\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Querying distinct log sources\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n from sqlalchemy import distinct\n\n sources = (\n session.query(distinct(TaskLogRecord.source))\n .filter(TaskLogRecord.task_id == task_id)\n .all()\n )\n result = [s[0] for s in sources]\n log_pl.reflect(\"Log sources retrieved\", payload={\"task_id\": task_id, \"sources\": result})\n return result\n finally:\n session.close()\n\n # [/DEF:get_sources:Function]\n" }, { "contract_id": "delete_logs_for_task", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 570, - "end_line": 594, + "start_line": 595, + "end_line": 622, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -46974,14 +47730,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:delete_logs_for_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for a specific task.\n # @PRE: task_id is a valid task ID.\n # @POST: All logs for the task are deleted.\n # @PARAM: task_id (str) - The task ID.\n # @SIDE_EFFECT: Deletes from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_task(self, task_id: str) -> None:\n with belief_scope(\n \"TaskLogPersistenceService.delete_logs_for_task\", f\"task_id={task_id}\"\n ):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n ).delete(synchronize_session=False)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_task:Function]\n" + "body": " # [DEF:delete_logs_for_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for a specific task.\n # @PRE: task_id is a valid task ID.\n # @POST: All logs for the task are deleted.\n # @PARAM: task_id (str) - The task ID.\n # @SIDE_EFFECT: Deletes from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_task(self, task_id: str) -> None:\n with belief_scope(\n \"TaskLogPersistenceService.delete_logs_for_task\", f\"task_id={task_id}\"\n ):\n log_pl.reason(\"Deleting logs for task\", payload={\"task_id\": task_id})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id == task_id\n ).delete(synchronize_session=False)\n session.commit()\n log_pl.reflect(\"Task logs deleted\", payload={\"task_id\": task_id})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to delete task logs\", error=str(e), payload={\"task_id\": task_id})\n logger.error(f\"Failed to delete logs for task {task_id}: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_task:Function]\n" }, { "contract_id": "delete_logs_for_tasks", "contract_type": "Function", "file_path": "backend/src/core/task_manager/persistence.py", - "start_line": 596, - "end_line": 620, + "start_line": 624, + "end_line": 651, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -47053,7 +47809,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:delete_logs_for_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for multiple tasks.\n # @PRE: task_ids is a list of task IDs.\n # @POST: All logs for the tasks are deleted.\n # @PARAM: task_ids (List[str]) - List of task IDs.\n # @SIDE_EFFECT: Deletes rows from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskLogPersistenceService.delete_logs_for_tasks\"):\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id.in_(task_ids)\n ).delete(synchronize_session=False)\n session.commit()\n except Exception as e:\n session.rollback()\n logger.error(f\"Failed to delete logs for tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_tasks:Function]\n" + "body": " # [DEF:delete_logs_for_tasks:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Delete all logs for multiple tasks.\n # @PRE: task_ids is a list of task IDs.\n # @POST: All logs for the tasks are deleted.\n # @PARAM: task_ids (List[str]) - List of task IDs.\n # @SIDE_EFFECT: Deletes rows from task_logs table.\n # @RELATION: [DEPENDS_ON] ->[TaskLogRecord]\n def delete_logs_for_tasks(self, task_ids: List[str]) -> None:\n if not task_ids:\n return\n with belief_scope(\"TaskLogPersistenceService.delete_logs_for_tasks\"):\n log_pl.reason(\"Deleting logs for multiple tasks\", payload={\"count\": len(task_ids)})\n session: Session = TasksSessionLocal()\n try:\n session.query(TaskLogRecord).filter(\n TaskLogRecord.task_id.in_(task_ids)\n ).delete(synchronize_session=False)\n session.commit()\n log_pl.reflect(\"Logs deleted for multiple tasks\", payload={\"count\": len(task_ids)})\n except Exception as e:\n session.rollback()\n log_pl.explore(\"Failed to delete logs for tasks\", error=str(e))\n logger.error(f\"Failed to delete logs for tasks: {e}\")\n finally:\n session.close()\n\n # [/DEF:delete_logs_for_tasks:Function]\n" }, { "contract_id": "TaskLoggerModule", @@ -47692,7 +48448,7 @@ "contract_type": "Module", "file_path": "backend/src/core/utils/async_network.py", "start_line": 1, - "end_line": 321, + "end_line": 323, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -47741,14 +48497,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:AsyncNetworkModule:Module]\n#\n# @COMPLEXITY: 5\n# @SEMANTICS: network, httpx, async, superset, authentication, cache\n# @PURPOSE: Provides async Superset API client with shared auth-token cache to avoid per-request re-login.\n# @LAYER: Infra\n# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login.\n# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.\n# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.\n# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.\n\n# [SECTION: IMPORTS]\nfrom typing import Optional, Dict, Any, Union\nimport asyncio\n\nimport httpx\n\nfrom ..logger import logger as app_logger, belief_scope\nfrom .network import (\n AuthenticationError,\n DashboardNotFoundError,\n NetworkError,\n PermissionDeniedError,\n SupersetAPIError,\n SupersetAuthCache,\n)\n# [/SECTION]\n\n\n# [DEF:AsyncAPIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [CALLS] ->[SupersetAuthCache.get]\n# @RELATION: [CALLS] ->[SupersetAuthCache.set]\nclass AsyncAPIClient:\n DEFAULT_TIMEOUT = 30\n _auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}\n\n # [DEF:AsyncAPIClient.__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initialize async API client for one environment.\n # @PRE: config contains base_url and auth payload.\n # @POST: Client is ready for async request/authentication flow.\n # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]\n # @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]\n # @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self._client = httpx.AsyncClient(\n verify=verify_ssl,\n timeout=httpx.Timeout(timeout),\n follow_redirects=True,\n )\n self._tokens: Dict[str, str] = {}\n self._authenticated = False\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n\n # [/DEF:AsyncAPIClient.__init__:Function]\n\n # [DEF:AsyncAPIClient._normalize_base_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Normalize base URL for Superset API root construction.\n # @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:AsyncAPIClient._normalize_base_url:Function]\n\n # [DEF:AsyncAPIClient._build_api_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Build full API URL from relative Superset endpoint.\n # @POST: Returns absolute URL for upstream request.\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:AsyncAPIClient._build_api_url:Function]\n\n # [DEF:AsyncAPIClient._get_auth_lock:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.\n # @POST: Returns stable asyncio.Lock instance.\n @classmethod\n def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:\n existing_lock = cls._auth_locks.get(cache_key)\n if existing_lock is not None:\n return existing_lock\n created_lock = asyncio.Lock()\n cls._auth_locks[cache_key] = created_lock\n return created_lock\n # [/DEF:AsyncAPIClient._get_auth_lock:Function]\n\n # [DEF:AsyncAPIClient.authenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticate against Superset and cache access/csrf tokens.\n # @POST: Client tokens are populated and reusable across requests.\n # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.\n # @DATA_CONTRACT: None -> Output[Dict[str, str]]\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]\n async def authenticate(self) -> Dict[str, str]:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this httpx client.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n )\n csrf_response.raise_for_status()\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n self._authenticated = True\n app_logger.info(\"[async_authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s\", self.base_url)\n return self._tokens\n except Exception as csrf_err:\n app_logger.warning(\"[async_authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s\", csrf_err)\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._tokens = {}\n self._authenticated = False\n\n auth_lock = self._get_auth_lock(self._auth_cache_key)\n async with auth_lock:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n self._authenticated = True\n app_logger.info(\"[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s\", self.base_url)\n return self._tokens\n\n with belief_scope(\"AsyncAPIClient.authenticate\"):\n app_logger.info(\"[async_authenticate][Enter] Authenticating to %s\", self.base_url)\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n response = await self._client.post(login_url, json=self.auth)\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {access_token}\"},\n )\n csrf_response.raise_for_status()\n\n self._tokens = {\n \"access_token\": access_token,\n \"csrf_token\": csrf_response.json()[\"result\"],\n }\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[async_authenticate][Exit] Authenticated successfully.\")\n return self._tokens\n except httpx.HTTPStatusError as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = exc.response.status_code if exc.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(\n f\"Environment unavailable during authentication (Status {status_code})\",\n status_code=status_code,\n ) from exc\n raise AuthenticationError(f\"Authentication failed: {exc}\") from exc\n except (httpx.HTTPError, KeyError) as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {exc}\") from exc\n # [/DEF:AsyncAPIClient.authenticate:Function]\n\n # [DEF:AsyncAPIClient.get_headers:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return authenticated Superset headers for async requests.\n # @POST: Headers include Authorization and CSRF tokens.\n # @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]\n async def get_headers(self) -> Dict[str, str]:\n if not self._authenticated:\n await self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\",\n }\n # [/DEF:AsyncAPIClient.get_headers:Function]\n\n # [DEF:AsyncAPIClient.request:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Perform one authenticated async Superset API request.\n # @POST: Returns JSON payload or raw httpx.Response when raw_response=true.\n # @SIDE_EFFECT: Performs network I/O.\n # @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]\n async def request(\n self,\n method: str,\n endpoint: str,\n headers: Optional[Dict[str, str]] = None,\n raw_response: bool = False,\n **kwargs,\n ) -> Union[httpx.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n request_headers = await self.get_headers()\n if headers:\n request_headers.update(headers)\n if \"allow_redirects\" in kwargs and \"follow_redirects\" not in kwargs:\n kwargs[\"follow_redirects\"] = bool(kwargs.pop(\"allow_redirects\"))\n\n try:\n response = await self._client.request(method, full_url, headers=request_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except httpx.HTTPStatusError as exc:\n if exc.response is not None and exc.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(exc, endpoint)\n except httpx.HTTPError as exc:\n self._handle_network_error(exc, full_url)\n # [/DEF:AsyncAPIClient.request:Function]\n\n # [DEF:AsyncAPIClient._handle_http_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.\n # @POST: Raises domain-specific exception for caller flow control.\n # @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception\n # @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]\n # @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]\n # @RELATION: [DEPENDS_ON] ->[SupersetAPIError]\n # @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]\n # @RELATION: [DEPENDS_ON] ->[AuthenticationError]\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_http_error\"):\n status_code = exc.response.status_code\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from exc\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from exc\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from exc\n if status_code == 403:\n raise PermissionDeniedError() from exc\n if status_code == 401:\n raise AuthenticationError() from exc\n raise SupersetAPIError(f\"API Error {status_code}: {exc.response.text}\") from exc\n # [/DEF:AsyncAPIClient._handle_http_error:Function]\n\n # [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n\n # [DEF:AsyncAPIClient._handle_network_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate generic httpx errors into NetworkError.\n # @POST: Raises NetworkError with URL context.\n # @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_network_error\"):\n if isinstance(exc, httpx.TimeoutException):\n message = \"Request timeout\"\n elif isinstance(exc, httpx.ConnectError):\n message = \"Connection error\"\n else:\n message = f\"Unknown network error: {exc}\"\n raise NetworkError(message, url=url) from exc\n # [/DEF:AsyncAPIClient._handle_network_error:Function]\n\n # [DEF:AsyncAPIClient.aclose:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Close underlying httpx client.\n # @POST: Client resources are released.\n # @SIDE_EFFECT: Closes network connections.\n # @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]\n async def aclose(self) -> None:\n await self._client.aclose()\n # [/DEF:AsyncAPIClient.aclose:Function]\n# [/DEF:AsyncAPIClient:Class]\n\n# [/DEF:AsyncNetworkModule:Module]\n" + "body": "# [DEF:AsyncNetworkModule:Module]\n#\n# @COMPLEXITY: 5\n# @SEMANTICS: network, httpx, async, superset, authentication, cache\n# @PURPOSE: Provides async Superset API client with shared auth-token cache to avoid per-request re-login.\n# @LAYER: Infra\n# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login.\n# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.\n# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.\n# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.\n\n# [SECTION: IMPORTS]\nfrom typing import Optional, Dict, Any, Union\nimport asyncio\n\nimport httpx\n\nfrom ..logger import logger as app_logger, belief_scope\nfrom ..cot_logger import MarkerLogger\nfrom .network import (\n AuthenticationError,\n DashboardNotFoundError,\n NetworkError,\n PermissionDeniedError,\n SupersetAPIError,\n SupersetAuthCache,\n)\n# [/SECTION]\n\nlog = MarkerLogger(\"AsyncNetwork\")\n\n# [DEF:AsyncAPIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [CALLS] ->[SupersetAuthCache.get]\n# @RELATION: [CALLS] ->[SupersetAuthCache.set]\nclass AsyncAPIClient:\n DEFAULT_TIMEOUT = 30\n _auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}\n\n # [DEF:AsyncAPIClient.__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initialize async API client for one environment.\n # @PRE: config contains base_url and auth payload.\n # @POST: Client is ready for async request/authentication flow.\n # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]\n # @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]\n # @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self._client = httpx.AsyncClient(\n verify=verify_ssl,\n timeout=httpx.Timeout(timeout),\n follow_redirects=True,\n )\n self._tokens: Dict[str, str] = {}\n self._authenticated = False\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n\n # [/DEF:AsyncAPIClient.__init__:Function]\n\n # [DEF:AsyncAPIClient._normalize_base_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Normalize base URL for Superset API root construction.\n # @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:AsyncAPIClient._normalize_base_url:Function]\n\n # [DEF:AsyncAPIClient._build_api_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Build full API URL from relative Superset endpoint.\n # @POST: Returns absolute URL for upstream request.\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:AsyncAPIClient._build_api_url:Function]\n\n # [DEF:AsyncAPIClient._get_auth_lock:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.\n # @POST: Returns stable asyncio.Lock instance.\n @classmethod\n def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:\n existing_lock = cls._auth_locks.get(cache_key)\n if existing_lock is not None:\n return existing_lock\n created_lock = asyncio.Lock()\n cls._auth_locks[cache_key] = created_lock\n return created_lock\n # [/DEF:AsyncAPIClient._get_auth_lock:Function]\n\n # [DEF:AsyncAPIClient.authenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticate against Superset and cache access/csrf tokens.\n # @POST: Client tokens are populated and reusable across requests.\n # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.\n # @DATA_CONTRACT: None -> Output[Dict[str, str]]\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]\n async def authenticate(self) -> Dict[str, str]:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this httpx client.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n )\n csrf_response.raise_for_status()\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n self._authenticated = True\n log.reason(\"Reused cached tokens + refreshed CSRF token\", payload={\"base_url\": self.base_url})\n return self._tokens\n except Exception as csrf_err:\n log.explore(\"CSRF refresh failed, performing full re-auth\", error=str(csrf_err))\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._tokens = {}\n self._authenticated = False\n\n auth_lock = self._get_auth_lock(self._auth_cache_key)\n async with auth_lock:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n self._authenticated = True\n log.reason(\"Reusing cached Superset auth tokens (after lock wait)\", payload={\"base_url\": self.base_url})\n return self._tokens\n\n with belief_scope(\"AsyncAPIClient.authenticate\"):\n log.reason(\"Performing full authentication\", payload={\"base_url\": self.base_url})\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n response = await self._client.post(login_url, json=self.auth)\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {access_token}\"},\n )\n csrf_response.raise_for_status()\n\n self._tokens = {\n \"access_token\": access_token,\n \"csrf_token\": csrf_response.json()[\"result\"],\n }\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reflect(\"Authenticated successfully\")\n return self._tokens\n except httpx.HTTPStatusError as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = exc.response.status_code if exc.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(\n f\"Environment unavailable during authentication (Status {status_code})\",\n status_code=status_code,\n ) from exc\n raise AuthenticationError(f\"Authentication failed: {exc}\") from exc\n except (httpx.HTTPError, KeyError) as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {exc}\") from exc\n # [/DEF:AsyncAPIClient.authenticate:Function]\n\n # [DEF:AsyncAPIClient.get_headers:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return authenticated Superset headers for async requests.\n # @POST: Headers include Authorization and CSRF tokens.\n # @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]\n async def get_headers(self) -> Dict[str, str]:\n if not self._authenticated:\n await self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\",\n }\n # [/DEF:AsyncAPIClient.get_headers:Function]\n\n # [DEF:AsyncAPIClient.request:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Perform one authenticated async Superset API request.\n # @POST: Returns JSON payload or raw httpx.Response when raw_response=true.\n # @SIDE_EFFECT: Performs network I/O.\n # @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]\n async def request(\n self,\n method: str,\n endpoint: str,\n headers: Optional[Dict[str, str]] = None,\n raw_response: bool = False,\n **kwargs,\n ) -> Union[httpx.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n request_headers = await self.get_headers()\n if headers:\n request_headers.update(headers)\n if \"allow_redirects\" in kwargs and \"follow_redirects\" not in kwargs:\n kwargs[\"follow_redirects\"] = bool(kwargs.pop(\"allow_redirects\"))\n\n try:\n response = await self._client.request(method, full_url, headers=request_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except httpx.HTTPStatusError as exc:\n if exc.response is not None and exc.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(exc, endpoint)\n except httpx.HTTPError as exc:\n self._handle_network_error(exc, full_url)\n # [/DEF:AsyncAPIClient.request:Function]\n\n # [DEF:AsyncAPIClient._handle_http_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.\n # @POST: Raises domain-specific exception for caller flow control.\n # @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception\n # @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]\n # @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]\n # @RELATION: [DEPENDS_ON] ->[SupersetAPIError]\n # @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]\n # @RELATION: [DEPENDS_ON] ->[AuthenticationError]\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_http_error\"):\n status_code = exc.response.status_code\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from exc\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from exc\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from exc\n if status_code == 403:\n raise PermissionDeniedError() from exc\n if status_code == 401:\n raise AuthenticationError() from exc\n raise SupersetAPIError(f\"API Error {status_code}: {exc.response.text}\") from exc\n # [/DEF:AsyncAPIClient._handle_http_error:Function]\n\n # [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n\n # [DEF:AsyncAPIClient._handle_network_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate generic httpx errors into NetworkError.\n # @POST: Raises NetworkError with URL context.\n # @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_network_error\"):\n if isinstance(exc, httpx.TimeoutException):\n message = \"Request timeout\"\n elif isinstance(exc, httpx.ConnectError):\n message = \"Connection error\"\n else:\n message = f\"Unknown network error: {exc}\"\n raise NetworkError(message, url=url) from exc\n # [/DEF:AsyncAPIClient._handle_network_error:Function]\n\n # [DEF:AsyncAPIClient.aclose:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Close underlying httpx client.\n # @POST: Client resources are released.\n # @SIDE_EFFECT: Closes network connections.\n # @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]\n async def aclose(self) -> None:\n await self._client.aclose()\n # [/DEF:AsyncAPIClient.aclose:Function]\n# [/DEF:AsyncAPIClient:Class]\n\n# [/DEF:AsyncNetworkModule:Module]\n" }, { "contract_id": "AsyncAPIClient", "contract_type": "Class", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 32, - "end_line": 319, + "start_line": 34, + "end_line": 321, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -47829,14 +48585,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:AsyncAPIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [CALLS] ->[SupersetAuthCache.get]\n# @RELATION: [CALLS] ->[SupersetAuthCache.set]\nclass AsyncAPIClient:\n DEFAULT_TIMEOUT = 30\n _auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}\n\n # [DEF:AsyncAPIClient.__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initialize async API client for one environment.\n # @PRE: config contains base_url and auth payload.\n # @POST: Client is ready for async request/authentication flow.\n # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]\n # @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]\n # @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self._client = httpx.AsyncClient(\n verify=verify_ssl,\n timeout=httpx.Timeout(timeout),\n follow_redirects=True,\n )\n self._tokens: Dict[str, str] = {}\n self._authenticated = False\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n\n # [/DEF:AsyncAPIClient.__init__:Function]\n\n # [DEF:AsyncAPIClient._normalize_base_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Normalize base URL for Superset API root construction.\n # @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:AsyncAPIClient._normalize_base_url:Function]\n\n # [DEF:AsyncAPIClient._build_api_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Build full API URL from relative Superset endpoint.\n # @POST: Returns absolute URL for upstream request.\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:AsyncAPIClient._build_api_url:Function]\n\n # [DEF:AsyncAPIClient._get_auth_lock:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.\n # @POST: Returns stable asyncio.Lock instance.\n @classmethod\n def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:\n existing_lock = cls._auth_locks.get(cache_key)\n if existing_lock is not None:\n return existing_lock\n created_lock = asyncio.Lock()\n cls._auth_locks[cache_key] = created_lock\n return created_lock\n # [/DEF:AsyncAPIClient._get_auth_lock:Function]\n\n # [DEF:AsyncAPIClient.authenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticate against Superset and cache access/csrf tokens.\n # @POST: Client tokens are populated and reusable across requests.\n # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.\n # @DATA_CONTRACT: None -> Output[Dict[str, str]]\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]\n async def authenticate(self) -> Dict[str, str]:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this httpx client.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n )\n csrf_response.raise_for_status()\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n self._authenticated = True\n app_logger.info(\"[async_authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s\", self.base_url)\n return self._tokens\n except Exception as csrf_err:\n app_logger.warning(\"[async_authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s\", csrf_err)\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._tokens = {}\n self._authenticated = False\n\n auth_lock = self._get_auth_lock(self._auth_cache_key)\n async with auth_lock:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n self._authenticated = True\n app_logger.info(\"[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s\", self.base_url)\n return self._tokens\n\n with belief_scope(\"AsyncAPIClient.authenticate\"):\n app_logger.info(\"[async_authenticate][Enter] Authenticating to %s\", self.base_url)\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n response = await self._client.post(login_url, json=self.auth)\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {access_token}\"},\n )\n csrf_response.raise_for_status()\n\n self._tokens = {\n \"access_token\": access_token,\n \"csrf_token\": csrf_response.json()[\"result\"],\n }\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[async_authenticate][Exit] Authenticated successfully.\")\n return self._tokens\n except httpx.HTTPStatusError as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = exc.response.status_code if exc.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(\n f\"Environment unavailable during authentication (Status {status_code})\",\n status_code=status_code,\n ) from exc\n raise AuthenticationError(f\"Authentication failed: {exc}\") from exc\n except (httpx.HTTPError, KeyError) as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {exc}\") from exc\n # [/DEF:AsyncAPIClient.authenticate:Function]\n\n # [DEF:AsyncAPIClient.get_headers:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return authenticated Superset headers for async requests.\n # @POST: Headers include Authorization and CSRF tokens.\n # @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]\n async def get_headers(self) -> Dict[str, str]:\n if not self._authenticated:\n await self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\",\n }\n # [/DEF:AsyncAPIClient.get_headers:Function]\n\n # [DEF:AsyncAPIClient.request:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Perform one authenticated async Superset API request.\n # @POST: Returns JSON payload or raw httpx.Response when raw_response=true.\n # @SIDE_EFFECT: Performs network I/O.\n # @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]\n async def request(\n self,\n method: str,\n endpoint: str,\n headers: Optional[Dict[str, str]] = None,\n raw_response: bool = False,\n **kwargs,\n ) -> Union[httpx.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n request_headers = await self.get_headers()\n if headers:\n request_headers.update(headers)\n if \"allow_redirects\" in kwargs and \"follow_redirects\" not in kwargs:\n kwargs[\"follow_redirects\"] = bool(kwargs.pop(\"allow_redirects\"))\n\n try:\n response = await self._client.request(method, full_url, headers=request_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except httpx.HTTPStatusError as exc:\n if exc.response is not None and exc.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(exc, endpoint)\n except httpx.HTTPError as exc:\n self._handle_network_error(exc, full_url)\n # [/DEF:AsyncAPIClient.request:Function]\n\n # [DEF:AsyncAPIClient._handle_http_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.\n # @POST: Raises domain-specific exception for caller flow control.\n # @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception\n # @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]\n # @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]\n # @RELATION: [DEPENDS_ON] ->[SupersetAPIError]\n # @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]\n # @RELATION: [DEPENDS_ON] ->[AuthenticationError]\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_http_error\"):\n status_code = exc.response.status_code\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from exc\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from exc\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from exc\n if status_code == 403:\n raise PermissionDeniedError() from exc\n if status_code == 401:\n raise AuthenticationError() from exc\n raise SupersetAPIError(f\"API Error {status_code}: {exc.response.text}\") from exc\n # [/DEF:AsyncAPIClient._handle_http_error:Function]\n\n # [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n\n # [DEF:AsyncAPIClient._handle_network_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate generic httpx errors into NetworkError.\n # @POST: Raises NetworkError with URL context.\n # @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_network_error\"):\n if isinstance(exc, httpx.TimeoutException):\n message = \"Request timeout\"\n elif isinstance(exc, httpx.ConnectError):\n message = \"Connection error\"\n else:\n message = f\"Unknown network error: {exc}\"\n raise NetworkError(message, url=url) from exc\n # [/DEF:AsyncAPIClient._handle_network_error:Function]\n\n # [DEF:AsyncAPIClient.aclose:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Close underlying httpx client.\n # @POST: Client resources are released.\n # @SIDE_EFFECT: Closes network connections.\n # @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]\n async def aclose(self) -> None:\n await self._client.aclose()\n # [/DEF:AsyncAPIClient.aclose:Function]\n# [/DEF:AsyncAPIClient:Class]\n" + "body": "# [DEF:AsyncAPIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [CALLS] ->[SupersetAuthCache.get]\n# @RELATION: [CALLS] ->[SupersetAuthCache.set]\nclass AsyncAPIClient:\n DEFAULT_TIMEOUT = 30\n _auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}\n\n # [DEF:AsyncAPIClient.__init__:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Initialize async API client for one environment.\n # @PRE: config contains base_url and auth payload.\n # @POST: Client is ready for async request/authentication flow.\n # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]\n # @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]\n # @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self._client = httpx.AsyncClient(\n verify=verify_ssl,\n timeout=httpx.Timeout(timeout),\n follow_redirects=True,\n )\n self._tokens: Dict[str, str] = {}\n self._authenticated = False\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n\n # [/DEF:AsyncAPIClient.__init__:Function]\n\n # [DEF:AsyncAPIClient._normalize_base_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Normalize base URL for Superset API root construction.\n # @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:AsyncAPIClient._normalize_base_url:Function]\n\n # [DEF:AsyncAPIClient._build_api_url:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Build full API URL from relative Superset endpoint.\n # @POST: Returns absolute URL for upstream request.\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:AsyncAPIClient._build_api_url:Function]\n\n # [DEF:AsyncAPIClient._get_auth_lock:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.\n # @POST: Returns stable asyncio.Lock instance.\n @classmethod\n def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:\n existing_lock = cls._auth_locks.get(cache_key)\n if existing_lock is not None:\n return existing_lock\n created_lock = asyncio.Lock()\n cls._auth_locks[cache_key] = created_lock\n return created_lock\n # [/DEF:AsyncAPIClient._get_auth_lock:Function]\n\n # [DEF:AsyncAPIClient.authenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticate against Superset and cache access/csrf tokens.\n # @POST: Client tokens are populated and reusable across requests.\n # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.\n # @DATA_CONTRACT: None -> Output[Dict[str, str]]\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]\n async def authenticate(self) -> Dict[str, str]:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this httpx client.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n )\n csrf_response.raise_for_status()\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n self._authenticated = True\n log.reason(\"Reused cached tokens + refreshed CSRF token\", payload={\"base_url\": self.base_url})\n return self._tokens\n except Exception as csrf_err:\n log.explore(\"CSRF refresh failed, performing full re-auth\", error=str(csrf_err))\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._tokens = {}\n self._authenticated = False\n\n auth_lock = self._get_auth_lock(self._auth_cache_key)\n async with auth_lock:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n self._authenticated = True\n log.reason(\"Reusing cached Superset auth tokens (after lock wait)\", payload={\"base_url\": self.base_url})\n return self._tokens\n\n with belief_scope(\"AsyncAPIClient.authenticate\"):\n log.reason(\"Performing full authentication\", payload={\"base_url\": self.base_url})\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n response = await self._client.post(login_url, json=self.auth)\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {access_token}\"},\n )\n csrf_response.raise_for_status()\n\n self._tokens = {\n \"access_token\": access_token,\n \"csrf_token\": csrf_response.json()[\"result\"],\n }\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reflect(\"Authenticated successfully\")\n return self._tokens\n except httpx.HTTPStatusError as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = exc.response.status_code if exc.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(\n f\"Environment unavailable during authentication (Status {status_code})\",\n status_code=status_code,\n ) from exc\n raise AuthenticationError(f\"Authentication failed: {exc}\") from exc\n except (httpx.HTTPError, KeyError) as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {exc}\") from exc\n # [/DEF:AsyncAPIClient.authenticate:Function]\n\n # [DEF:AsyncAPIClient.get_headers:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return authenticated Superset headers for async requests.\n # @POST: Headers include Authorization and CSRF tokens.\n # @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]\n async def get_headers(self) -> Dict[str, str]:\n if not self._authenticated:\n await self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\",\n }\n # [/DEF:AsyncAPIClient.get_headers:Function]\n\n # [DEF:AsyncAPIClient.request:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Perform one authenticated async Superset API request.\n # @POST: Returns JSON payload or raw httpx.Response when raw_response=true.\n # @SIDE_EFFECT: Performs network I/O.\n # @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]\n # @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]\n async def request(\n self,\n method: str,\n endpoint: str,\n headers: Optional[Dict[str, str]] = None,\n raw_response: bool = False,\n **kwargs,\n ) -> Union[httpx.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n request_headers = await self.get_headers()\n if headers:\n request_headers.update(headers)\n if \"allow_redirects\" in kwargs and \"follow_redirects\" not in kwargs:\n kwargs[\"follow_redirects\"] = bool(kwargs.pop(\"allow_redirects\"))\n\n try:\n response = await self._client.request(method, full_url, headers=request_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except httpx.HTTPStatusError as exc:\n if exc.response is not None and exc.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(exc, endpoint)\n except httpx.HTTPError as exc:\n self._handle_network_error(exc, full_url)\n # [/DEF:AsyncAPIClient.request:Function]\n\n # [DEF:AsyncAPIClient._handle_http_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.\n # @POST: Raises domain-specific exception for caller flow control.\n # @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception\n # @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]\n # @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]\n # @RELATION: [DEPENDS_ON] ->[SupersetAPIError]\n # @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]\n # @RELATION: [DEPENDS_ON] ->[AuthenticationError]\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_http_error\"):\n status_code = exc.response.status_code\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from exc\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from exc\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from exc\n if status_code == 403:\n raise PermissionDeniedError() from exc\n if status_code == 401:\n raise AuthenticationError() from exc\n raise SupersetAPIError(f\"API Error {status_code}: {exc.response.text}\") from exc\n # [/DEF:AsyncAPIClient._handle_http_error:Function]\n\n # [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function]\n\n # [DEF:AsyncAPIClient._handle_network_error:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Translate generic httpx errors into NetworkError.\n # @POST: Raises NetworkError with URL context.\n # @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError\n # @RELATION: [DEPENDS_ON] ->[NetworkError]\n def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:\n with belief_scope(\"AsyncAPIClient._handle_network_error\"):\n if isinstance(exc, httpx.TimeoutException):\n message = \"Request timeout\"\n elif isinstance(exc, httpx.ConnectError):\n message = \"Connection error\"\n else:\n message = f\"Unknown network error: {exc}\"\n raise NetworkError(message, url=url) from exc\n # [/DEF:AsyncAPIClient._handle_network_error:Function]\n\n # [DEF:AsyncAPIClient.aclose:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Close underlying httpx client.\n # @POST: Client resources are released.\n # @SIDE_EFFECT: Closes network connections.\n # @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]\n async def aclose(self) -> None:\n await self._client.aclose()\n # [/DEF:AsyncAPIClient.aclose:Function]\n# [/DEF:AsyncAPIClient:Class]\n" }, { "contract_id": "AsyncAPIClient.__init__", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 42, - "end_line": 68, + "start_line": 44, + "end_line": 70, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -47930,8 +48686,8 @@ "contract_id": "AsyncAPIClient._normalize_base_url", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 70, - "end_line": 79, + "start_line": 72, + "end_line": 81, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -47958,8 +48714,8 @@ "contract_id": "AsyncAPIClient._build_api_url", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 81, - "end_line": 94, + "start_line": 83, + "end_line": 96, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -47986,8 +48742,8 @@ "contract_id": "AsyncAPIClient._get_auth_lock", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 96, - "end_line": 108, + "start_line": 98, + "end_line": 110, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48014,8 +48770,8 @@ "contract_id": "AsyncAPIClient.authenticate", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 110, - "end_line": 188, + "start_line": 112, + "end_line": 190, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -48126,14 +48882,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:AsyncAPIClient.authenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticate against Superset and cache access/csrf tokens.\n # @POST: Client tokens are populated and reusable across requests.\n # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.\n # @DATA_CONTRACT: None -> Output[Dict[str, str]]\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]\n async def authenticate(self) -> Dict[str, str]:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this httpx client.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n )\n csrf_response.raise_for_status()\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n self._authenticated = True\n app_logger.info(\"[async_authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s\", self.base_url)\n return self._tokens\n except Exception as csrf_err:\n app_logger.warning(\"[async_authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s\", csrf_err)\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._tokens = {}\n self._authenticated = False\n\n auth_lock = self._get_auth_lock(self._auth_cache_key)\n async with auth_lock:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n self._authenticated = True\n app_logger.info(\"[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s\", self.base_url)\n return self._tokens\n\n with belief_scope(\"AsyncAPIClient.authenticate\"):\n app_logger.info(\"[async_authenticate][Enter] Authenticating to %s\", self.base_url)\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n response = await self._client.post(login_url, json=self.auth)\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {access_token}\"},\n )\n csrf_response.raise_for_status()\n\n self._tokens = {\n \"access_token\": access_token,\n \"csrf_token\": csrf_response.json()[\"result\"],\n }\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[async_authenticate][Exit] Authenticated successfully.\")\n return self._tokens\n except httpx.HTTPStatusError as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = exc.response.status_code if exc.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(\n f\"Environment unavailable during authentication (Status {status_code})\",\n status_code=status_code,\n ) from exc\n raise AuthenticationError(f\"Authentication failed: {exc}\") from exc\n except (httpx.HTTPError, KeyError) as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {exc}\") from exc\n # [/DEF:AsyncAPIClient.authenticate:Function]\n" + "body": " # [DEF:AsyncAPIClient.authenticate:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Authenticate against Superset and cache access/csrf tokens.\n # @POST: Client tokens are populated and reusable across requests.\n # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.\n # @DATA_CONTRACT: None -> Output[Dict[str, str]]\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]\n async def authenticate(self) -> Dict[str, str]:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this httpx client.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n )\n csrf_response.raise_for_status()\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n self._authenticated = True\n log.reason(\"Reused cached tokens + refreshed CSRF token\", payload={\"base_url\": self.base_url})\n return self._tokens\n except Exception as csrf_err:\n log.explore(\"CSRF refresh failed, performing full re-auth\", error=str(csrf_err))\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._tokens = {}\n self._authenticated = False\n\n auth_lock = self._get_auth_lock(self._auth_cache_key)\n async with auth_lock:\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n self._authenticated = True\n log.reason(\"Reusing cached Superset auth tokens (after lock wait)\", payload={\"base_url\": self.base_url})\n return self._tokens\n\n with belief_scope(\"AsyncAPIClient.authenticate\"):\n log.reason(\"Performing full authentication\", payload={\"base_url\": self.base_url})\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n response = await self._client.post(login_url, json=self.auth)\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = await self._client.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {access_token}\"},\n )\n csrf_response.raise_for_status()\n\n self._tokens = {\n \"access_token\": access_token,\n \"csrf_token\": csrf_response.json()[\"result\"],\n }\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reflect(\"Authenticated successfully\")\n return self._tokens\n except httpx.HTTPStatusError as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = exc.response.status_code if exc.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(\n f\"Environment unavailable during authentication (Status {status_code})\",\n status_code=status_code,\n ) from exc\n raise AuthenticationError(f\"Authentication failed: {exc}\") from exc\n except (httpx.HTTPError, KeyError) as exc:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {exc}\") from exc\n # [/DEF:AsyncAPIClient.authenticate:Function]\n" }, { "contract_id": "AsyncAPIClient.get_headers", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 190, - "end_line": 204, + "start_line": 192, + "end_line": 206, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -48184,8 +48940,8 @@ "contract_id": "AsyncAPIClient.request", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 206, - "end_line": 241, + "start_line": 208, + "end_line": 243, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -48292,8 +49048,8 @@ "contract_id": "AsyncAPIClient._handle_http_error", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 243, - "end_line": 273, + "start_line": 245, + "end_line": 275, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -48469,8 +49225,8 @@ "contract_id": "AsyncAPIClient._is_dashboard_endpoint", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 275, - "end_line": 291, + "start_line": 277, + "end_line": 293, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -48497,8 +49253,8 @@ "contract_id": "AsyncAPIClient._handle_network_error", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 293, - "end_line": 308, + "start_line": 295, + "end_line": 310, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -48559,8 +49315,8 @@ "contract_id": "AsyncAPIClient.aclose", "contract_type": "Function", "file_path": "backend/src/core/utils/async_network.py", - "start_line": 310, - "end_line": 318, + "start_line": 312, + "end_line": 320, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -48622,7 +49378,7 @@ "contract_type": "Module", "file_path": "backend/src/core/utils/dataset_mapper.py", "start_line": 1, - "end_line": 237, + "end_line": 242, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48675,14 +49431,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:DatasetMapperModule:Module]\n#\n# @SEMANTICS: dataset, mapping, postgresql, xlsx, superset\n# @PURPOSE: Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> backend.core.superset_client\n# @RELATION: DEPENDS_ON -> pandas\n# @RELATION: DEPENDS_ON -> psycopg2\n# @PUBLIC_API: DatasetMapper\n\n# [SECTION: IMPORTS]\nimport pandas as pd # type: ignore\nimport psycopg2 # type: ignore\nfrom typing import Dict, Optional, Any\nfrom ..logger import logger as app_logger, belief_scope\n# [/SECTION]\n\n# [DEF:DatasetMapper:Class]\n# @PURPOSE: Класс для меппинга и обновления verbose_map в датасетах Superset.\nclass DatasetMapper:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapper.\n # @POST: Объект DatasetMapper инициализирован.\n def __init__(self):\n pass\n # [/DEF:__init__:Function]\n\n # [DEF:get_postgres_comments:Function]\n # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.\n # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).\n # @PRE: table_name и table_schema должны быть строками.\n # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.\n # @THROW: Exception - При ошибках подключения или выполнения запроса к БД.\n # @PARAM: db_config (Dict) - Конфигурация для подключения к БД.\n # @PARAM: table_name (str) - Имя таблицы.\n # @PARAM: table_schema (str) - Схема таблицы.\n # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.\n def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:\n with belief_scope(\"Fetch comments from PostgreSQL\"):\n app_logger.info(\"[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.\", table_schema, table_name)\n query = f\"\"\"\n SELECT \n cols.column_name,\n CASE \n WHEN pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ) LIKE '%|%' THEN \n split_part(\n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ), \n '|', \n 1\n )\n ELSE \n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n )\n END AS column_comment\n FROM \n information_schema.columns cols \n WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';\n \"\"\"\n comments = {}\n try:\n with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:\n cursor.execute(query)\n for row in cursor.fetchall():\n if row[1]:\n comments[row[0]] = row[1]\n app_logger.info(\"[get_postgres_comments][Success] Fetched %d comments.\", len(comments))\n except Exception as e:\n app_logger.error(\"[get_postgres_comments][Failure] %s\", e, exc_info=True)\n raise\n return comments\n # [/DEF:get_postgres_comments:Function]\n\n # [DEF:load_excel_mappings:Function]\n # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.\n # @PRE: file_path должен указывать на существующий XLSX файл.\n # @POST: Возвращается словарь с меппингами из файла.\n # @THROW: Exception - При ошибках чтения файла или парсинга.\n # @PARAM: file_path (str) - Путь к XLSX файлу.\n # @RETURN: Dict[str, str] - Словарь с меппингами.\n def load_excel_mappings(self, file_path: str) -> Dict[str, str]:\n with belief_scope(\"Load mappings from Excel\"):\n app_logger.info(\"[load_excel_mappings][Enter] Loading mappings from %s.\", file_path)\n try:\n df = pd.read_excel(file_path)\n mappings = df.set_index('column_name')['verbose_name'].to_dict()\n app_logger.info(\"[load_excel_mappings][Success] Loaded %d mappings.\", len(mappings))\n return mappings\n except Exception as e:\n app_logger.error(\"[load_excel_mappings][Failure] %s\", e, exc_info=True)\n raise\n # [/DEF:load_excel_mappings:Function]\n\n # [DEF:run_mapping:Function]\n # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.\n # @PRE: superset_client должен быть авторизован.\n # @PRE: dataset_id должен быть существующим ID в Superset.\n # @POST: Если найдены изменения, датасет в Superset обновлен через API.\n # @RELATION: CALLS -> self.get_postgres_comments\n # @RELATION: CALLS -> self.load_excel_mappings\n # @RELATION: CALLS -> superset_client.get_dataset\n # @RELATION: CALLS -> superset_client.update_dataset\n # @PARAM: superset_client (Any) - Клиент Superset.\n # @PARAM: dataset_id (int) - ID датасета для обновления.\n # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').\n # @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.\n # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.\n # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.\n # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.\n def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):\n with belief_scope(f\"Run dataset mapping for ID {dataset_id}\"):\n app_logger.info(\"[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.\", dataset_id, source)\n mappings: Dict[str, str] = {}\n \n try:\n if source in ['postgres', 'both']:\n assert postgres_config and table_name and table_schema, \"Postgres config is required.\"\n mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))\n if source in ['excel', 'both']:\n assert excel_path, \"Excel path is required.\"\n mappings.update(self.load_excel_mappings(excel_path))\n if source not in ['postgres', 'excel', 'both']:\n app_logger.error(\"[run_mapping][Failure] Invalid source: %s.\", source)\n return\n\n dataset_response = superset_client.get_dataset(dataset_id)\n dataset_data = dataset_response['result']\n \n original_columns = dataset_data.get('columns', [])\n updated_columns = []\n changes_made = False\n\n for column in original_columns:\n col_name = column.get('column_name')\n \n new_column = {\n \"column_name\": col_name,\n \"id\": column.get(\"id\"),\n \"advanced_data_type\": column.get(\"advanced_data_type\"),\n \"description\": column.get(\"description\"),\n \"expression\": column.get(\"expression\"),\n \"extra\": column.get(\"extra\"),\n \"filterable\": column.get(\"filterable\"),\n \"groupby\": column.get(\"groupby\"),\n \"is_active\": column.get(\"is_active\"),\n \"is_dttm\": column.get(\"is_dttm\"),\n \"python_date_format\": column.get(\"python_date_format\"),\n \"type\": column.get(\"type\"),\n \"uuid\": column.get(\"uuid\"),\n \"verbose_name\": column.get(\"verbose_name\"),\n }\n \n new_column = {k: v for k, v in new_column.items() if v is not None}\n\n if col_name in mappings:\n mapping_value = mappings[col_name]\n if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:\n new_column['verbose_name'] = mapping_value\n changes_made = True\n \n updated_columns.append(new_column)\n\n updated_metrics = []\n for metric in dataset_data.get(\"metrics\", []):\n new_metric = {\n \"id\": metric.get(\"id\"),\n \"metric_name\": metric.get(\"metric_name\"),\n \"expression\": metric.get(\"expression\"),\n \"verbose_name\": metric.get(\"verbose_name\"),\n \"description\": metric.get(\"description\"),\n \"d3format\": metric.get(\"d3format\"),\n \"currency\": metric.get(\"currency\"),\n \"extra\": metric.get(\"extra\"),\n \"warning_text\": metric.get(\"warning_text\"),\n \"metric_type\": metric.get(\"metric_type\"),\n \"uuid\": metric.get(\"uuid\"),\n }\n updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})\n\n if changes_made:\n payload_for_update = {\n \"database_id\": dataset_data.get(\"database\", {}).get(\"id\"),\n \"table_name\": dataset_data.get(\"table_name\"),\n \"schema\": dataset_data.get(\"schema\"),\n \"columns\": updated_columns,\n \"owners\": [owner[\"id\"] for owner in dataset_data.get(\"owners\", [])],\n \"metrics\": updated_metrics,\n \"extra\": dataset_data.get(\"extra\"),\n \"description\": dataset_data.get(\"description\"),\n \"sql\": dataset_data.get(\"sql\"),\n \"cache_timeout\": dataset_data.get(\"cache_timeout\"),\n \"catalog\": dataset_data.get(\"catalog\"),\n \"default_endpoint\": dataset_data.get(\"default_endpoint\"),\n \"external_url\": dataset_data.get(\"external_url\"),\n \"fetch_values_predicate\": dataset_data.get(\"fetch_values_predicate\"),\n \"filter_select_enabled\": dataset_data.get(\"filter_select_enabled\"),\n \"is_managed_externally\": dataset_data.get(\"is_managed_externally\"),\n \"is_sqllab_view\": dataset_data.get(\"is_sqllab_view\"),\n \"main_dttm_col\": dataset_data.get(\"main_dttm_col\"),\n \"normalize_columns\": dataset_data.get(\"normalize_columns\"),\n \"offset\": dataset_data.get(\"offset\"),\n \"template_params\": dataset_data.get(\"template_params\"),\n }\n \n payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}\n\n superset_client.update_dataset(dataset_id, payload_for_update)\n app_logger.info(\"[run_mapping][Success] Dataset %d columns' verbose_name updated.\", dataset_id)\n else:\n app_logger.info(\"[run_mapping][State] No changes in columns' verbose_name, skipping update.\")\n\n except (AssertionError, FileNotFoundError, Exception) as e:\n app_logger.error(\"[run_mapping][Failure] %s\", e, exc_info=True)\n return\n # [/DEF:run_mapping:Function]\n# [/DEF:DatasetMapper:Class]\n\n# [/DEF:DatasetMapperModule:Module]\n" + "body": "# [DEF:DatasetMapperModule:Module]\n#\n# @SEMANTICS: dataset, mapping, postgresql, xlsx, superset\n# @PURPOSE: Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> backend.core.superset_client\n# @RELATION: DEPENDS_ON -> pandas\n# @RELATION: DEPENDS_ON -> psycopg2\n# @PUBLIC_API: DatasetMapper\n\n# [SECTION: IMPORTS]\nimport pandas as pd # type: ignore\nimport psycopg2 # type: ignore\nfrom typing import Dict, Optional, Any\nfrom ..logger import belief_scope\nfrom ..cot_logger import MarkerLogger\n# [/SECTION]\n\nlog = MarkerLogger(\"DatasetMapper\")\n\n# [DEF:DatasetMapper:Class]\n# @PURPOSE: Класс для меппинга и обновления verbose_map в датасетах Superset.\nclass DatasetMapper:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapper.\n # @POST: Объект DatasetMapper инициализирован.\n def __init__(self):\n pass\n # [/DEF:__init__:Function]\n\n # [DEF:get_postgres_comments:Function]\n # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.\n # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).\n # @PRE: table_name и table_schema должны быть строками.\n # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.\n # @THROW: Exception - При ошибках подключения или выполнения запроса к БД.\n # @PARAM: db_config (Dict) - Конфигурация для подключения к БД.\n # @PARAM: table_name (str) - Имя таблицы.\n # @PARAM: table_schema (str) - Схема таблицы.\n # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.\n def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:\n with belief_scope(\"Fetch comments from PostgreSQL\"):\n log.reason(\"Fetching comments from PostgreSQL\", payload={\"table_schema\": table_schema, \"table_name\": table_name})\n query = f\"\"\"\n SELECT \n cols.column_name,\n CASE \n WHEN pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ) LIKE '%|%' THEN \n split_part(\n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ), \n '|', \n 1\n )\n ELSE \n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n )\n END AS column_comment\n FROM \n information_schema.columns cols \n WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';\n \"\"\"\n comments = {}\n try:\n with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:\n cursor.execute(query)\n for row in cursor.fetchall():\n if row[1]:\n comments[row[0]] = row[1]\n log.reflect(\"PostgreSQL comments fetched\", payload={\"count\": len(comments)})\n except Exception as e:\n log.explore(\"Failed to fetch PostgreSQL comments\", payload={\"table_schema\": table_schema, \"table_name\": table_name}, error=str(e))\n raise\n return comments\n # [/DEF:get_postgres_comments:Function]\n\n # [DEF:load_excel_mappings:Function]\n # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.\n # @PRE: file_path должен указывать на существующий XLSX файл.\n # @POST: Возвращается словарь с меппингами из файла.\n # @THROW: Exception - При ошибках чтения файла или парсинга.\n # @PARAM: file_path (str) - Путь к XLSX файлу.\n # @RETURN: Dict[str, str] - Словарь с меппингами.\n def load_excel_mappings(self, file_path: str) -> Dict[str, str]:\n with belief_scope(\"Load mappings from Excel\"):\n log.reason(\"Loading mappings from Excel\", payload={\"file_path\": file_path})\n try:\n df = pd.read_excel(file_path)\n mappings = df.set_index('column_name')['verbose_name'].to_dict()\n log.reflect(\"Excel mappings loaded\", payload={\"count\": len(mappings)})\n return mappings\n except Exception as e:\n log.explore(\"Failed to load Excel mappings\", payload={\"file_path\": file_path}, error=str(e))\n raise\n # [/DEF:load_excel_mappings:Function]\n\n # [DEF:run_mapping:Function]\n # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.\n # @PRE: superset_client должен быть авторизован.\n # @PRE: dataset_id должен быть существующим ID в Superset.\n # @POST: Если найдены изменения, датасет в Superset обновлен через API.\n # @RELATION: CALLS -> self.get_postgres_comments\n # @RELATION: CALLS -> self.load_excel_mappings\n # @RELATION: CALLS -> superset_client.get_dataset\n # @RELATION: CALLS -> superset_client.update_dataset\n # @PARAM: superset_client (Any) - Клиент Superset.\n # @PARAM: dataset_id (int) - ID датасета для обновления.\n # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').\n # @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.\n # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.\n # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.\n # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.\n def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):\n with belief_scope(f\"Run dataset mapping for ID {dataset_id}\"):\n log.reason(\"Starting dataset mapping\", payload={\"dataset_id\": dataset_id, \"source\": source})\n mappings: Dict[str, str] = {}\n \n try:\n if source in ['postgres', 'both']:\n if not (postgres_config and table_name and table_schema):\n raise ValueError(\"Postgres config is required.\")\n mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))\n if source in ['excel', 'both']:\n if not excel_path:\n raise ValueError(\"Excel path is required.\")\n mappings.update(self.load_excel_mappings(excel_path))\n if source not in ['postgres', 'excel', 'both']:\n log.explore(\"Invalid mapping source\", payload={\"source\": source}, error=f\"Invalid source: {source}\")\n return\n\n dataset_response = superset_client.get_dataset(dataset_id)\n dataset_data = dataset_response['result']\n \n original_columns = dataset_data.get('columns', [])\n updated_columns = []\n changes_made = False\n\n for column in original_columns:\n col_name = column.get('column_name')\n \n new_column = {\n \"column_name\": col_name,\n \"id\": column.get(\"id\"),\n \"advanced_data_type\": column.get(\"advanced_data_type\"),\n \"description\": column.get(\"description\"),\n \"expression\": column.get(\"expression\"),\n \"extra\": column.get(\"extra\"),\n \"filterable\": column.get(\"filterable\"),\n \"groupby\": column.get(\"groupby\"),\n \"is_active\": column.get(\"is_active\"),\n \"is_dttm\": column.get(\"is_dttm\"),\n \"python_date_format\": column.get(\"python_date_format\"),\n \"type\": column.get(\"type\"),\n \"uuid\": column.get(\"uuid\"),\n \"verbose_name\": column.get(\"verbose_name\"),\n }\n \n new_column = {k: v for k, v in new_column.items() if v is not None}\n\n if col_name in mappings:\n mapping_value = mappings[col_name]\n if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:\n new_column['verbose_name'] = mapping_value\n changes_made = True\n \n updated_columns.append(new_column)\n\n updated_metrics = []\n for metric in dataset_data.get(\"metrics\", []):\n new_metric = {\n \"id\": metric.get(\"id\"),\n \"metric_name\": metric.get(\"metric_name\"),\n \"expression\": metric.get(\"expression\"),\n \"verbose_name\": metric.get(\"verbose_name\"),\n \"description\": metric.get(\"description\"),\n \"d3format\": metric.get(\"d3format\"),\n \"currency\": metric.get(\"currency\"),\n \"extra\": metric.get(\"extra\"),\n \"warning_text\": metric.get(\"warning_text\"),\n \"metric_type\": metric.get(\"metric_type\"),\n \"uuid\": metric.get(\"uuid\"),\n }\n updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})\n\n if changes_made:\n payload_for_update = {\n \"database_id\": dataset_data.get(\"database\", {}).get(\"id\"),\n \"table_name\": dataset_data.get(\"table_name\"),\n \"schema\": dataset_data.get(\"schema\"),\n \"columns\": updated_columns,\n \"owners\": [owner[\"id\"] for owner in dataset_data.get(\"owners\", [])],\n \"metrics\": updated_metrics,\n \"extra\": dataset_data.get(\"extra\"),\n \"description\": dataset_data.get(\"description\"),\n \"sql\": dataset_data.get(\"sql\"),\n \"cache_timeout\": dataset_data.get(\"cache_timeout\"),\n \"catalog\": dataset_data.get(\"catalog\"),\n \"default_endpoint\": dataset_data.get(\"default_endpoint\"),\n \"external_url\": dataset_data.get(\"external_url\"),\n \"fetch_values_predicate\": dataset_data.get(\"fetch_values_predicate\"),\n \"filter_select_enabled\": dataset_data.get(\"filter_select_enabled\"),\n \"is_managed_externally\": dataset_data.get(\"is_managed_externally\"),\n \"is_sqllab_view\": dataset_data.get(\"is_sqllab_view\"),\n \"main_dttm_col\": dataset_data.get(\"main_dttm_col\"),\n \"normalize_columns\": dataset_data.get(\"normalize_columns\"),\n \"offset\": dataset_data.get(\"offset\"),\n \"template_params\": dataset_data.get(\"template_params\"),\n }\n \n payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}\n\n superset_client.update_dataset(dataset_id, payload_for_update)\n log.reflect(\"Dataset verbose_name updated\", payload={\"dataset_id\": dataset_id})\n else:\n log.reflect(\"No changes in verbose_name, skipping update\", payload={\"dataset_id\": dataset_id})\n\n except (FileNotFoundError, Exception) as e:\n log.explore(\"Dataset mapping failed\", payload={\"dataset_id\": dataset_id, \"source\": source}, error=str(e))\n return\n # [/DEF:run_mapping:Function]\n# [/DEF:DatasetMapper:Class]\n\n# [/DEF:DatasetMapperModule:Module]\n" }, { "contract_id": "DatasetMapper", "contract_type": "Class", "file_path": "backend/src/core/utils/dataset_mapper.py", - "start_line": 18, - "end_line": 235, + "start_line": 21, + "end_line": 240, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48691,14 +49447,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:DatasetMapper:Class]\n# @PURPOSE: Класс для меппинга и обновления verbose_map в датасетах Superset.\nclass DatasetMapper:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapper.\n # @POST: Объект DatasetMapper инициализирован.\n def __init__(self):\n pass\n # [/DEF:__init__:Function]\n\n # [DEF:get_postgres_comments:Function]\n # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.\n # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).\n # @PRE: table_name и table_schema должны быть строками.\n # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.\n # @THROW: Exception - При ошибках подключения или выполнения запроса к БД.\n # @PARAM: db_config (Dict) - Конфигурация для подключения к БД.\n # @PARAM: table_name (str) - Имя таблицы.\n # @PARAM: table_schema (str) - Схема таблицы.\n # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.\n def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:\n with belief_scope(\"Fetch comments from PostgreSQL\"):\n app_logger.info(\"[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.\", table_schema, table_name)\n query = f\"\"\"\n SELECT \n cols.column_name,\n CASE \n WHEN pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ) LIKE '%|%' THEN \n split_part(\n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ), \n '|', \n 1\n )\n ELSE \n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n )\n END AS column_comment\n FROM \n information_schema.columns cols \n WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';\n \"\"\"\n comments = {}\n try:\n with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:\n cursor.execute(query)\n for row in cursor.fetchall():\n if row[1]:\n comments[row[0]] = row[1]\n app_logger.info(\"[get_postgres_comments][Success] Fetched %d comments.\", len(comments))\n except Exception as e:\n app_logger.error(\"[get_postgres_comments][Failure] %s\", e, exc_info=True)\n raise\n return comments\n # [/DEF:get_postgres_comments:Function]\n\n # [DEF:load_excel_mappings:Function]\n # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.\n # @PRE: file_path должен указывать на существующий XLSX файл.\n # @POST: Возвращается словарь с меппингами из файла.\n # @THROW: Exception - При ошибках чтения файла или парсинга.\n # @PARAM: file_path (str) - Путь к XLSX файлу.\n # @RETURN: Dict[str, str] - Словарь с меппингами.\n def load_excel_mappings(self, file_path: str) -> Dict[str, str]:\n with belief_scope(\"Load mappings from Excel\"):\n app_logger.info(\"[load_excel_mappings][Enter] Loading mappings from %s.\", file_path)\n try:\n df = pd.read_excel(file_path)\n mappings = df.set_index('column_name')['verbose_name'].to_dict()\n app_logger.info(\"[load_excel_mappings][Success] Loaded %d mappings.\", len(mappings))\n return mappings\n except Exception as e:\n app_logger.error(\"[load_excel_mappings][Failure] %s\", e, exc_info=True)\n raise\n # [/DEF:load_excel_mappings:Function]\n\n # [DEF:run_mapping:Function]\n # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.\n # @PRE: superset_client должен быть авторизован.\n # @PRE: dataset_id должен быть существующим ID в Superset.\n # @POST: Если найдены изменения, датасет в Superset обновлен через API.\n # @RELATION: CALLS -> self.get_postgres_comments\n # @RELATION: CALLS -> self.load_excel_mappings\n # @RELATION: CALLS -> superset_client.get_dataset\n # @RELATION: CALLS -> superset_client.update_dataset\n # @PARAM: superset_client (Any) - Клиент Superset.\n # @PARAM: dataset_id (int) - ID датасета для обновления.\n # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').\n # @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.\n # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.\n # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.\n # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.\n def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):\n with belief_scope(f\"Run dataset mapping for ID {dataset_id}\"):\n app_logger.info(\"[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.\", dataset_id, source)\n mappings: Dict[str, str] = {}\n \n try:\n if source in ['postgres', 'both']:\n assert postgres_config and table_name and table_schema, \"Postgres config is required.\"\n mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))\n if source in ['excel', 'both']:\n assert excel_path, \"Excel path is required.\"\n mappings.update(self.load_excel_mappings(excel_path))\n if source not in ['postgres', 'excel', 'both']:\n app_logger.error(\"[run_mapping][Failure] Invalid source: %s.\", source)\n return\n\n dataset_response = superset_client.get_dataset(dataset_id)\n dataset_data = dataset_response['result']\n \n original_columns = dataset_data.get('columns', [])\n updated_columns = []\n changes_made = False\n\n for column in original_columns:\n col_name = column.get('column_name')\n \n new_column = {\n \"column_name\": col_name,\n \"id\": column.get(\"id\"),\n \"advanced_data_type\": column.get(\"advanced_data_type\"),\n \"description\": column.get(\"description\"),\n \"expression\": column.get(\"expression\"),\n \"extra\": column.get(\"extra\"),\n \"filterable\": column.get(\"filterable\"),\n \"groupby\": column.get(\"groupby\"),\n \"is_active\": column.get(\"is_active\"),\n \"is_dttm\": column.get(\"is_dttm\"),\n \"python_date_format\": column.get(\"python_date_format\"),\n \"type\": column.get(\"type\"),\n \"uuid\": column.get(\"uuid\"),\n \"verbose_name\": column.get(\"verbose_name\"),\n }\n \n new_column = {k: v for k, v in new_column.items() if v is not None}\n\n if col_name in mappings:\n mapping_value = mappings[col_name]\n if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:\n new_column['verbose_name'] = mapping_value\n changes_made = True\n \n updated_columns.append(new_column)\n\n updated_metrics = []\n for metric in dataset_data.get(\"metrics\", []):\n new_metric = {\n \"id\": metric.get(\"id\"),\n \"metric_name\": metric.get(\"metric_name\"),\n \"expression\": metric.get(\"expression\"),\n \"verbose_name\": metric.get(\"verbose_name\"),\n \"description\": metric.get(\"description\"),\n \"d3format\": metric.get(\"d3format\"),\n \"currency\": metric.get(\"currency\"),\n \"extra\": metric.get(\"extra\"),\n \"warning_text\": metric.get(\"warning_text\"),\n \"metric_type\": metric.get(\"metric_type\"),\n \"uuid\": metric.get(\"uuid\"),\n }\n updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})\n\n if changes_made:\n payload_for_update = {\n \"database_id\": dataset_data.get(\"database\", {}).get(\"id\"),\n \"table_name\": dataset_data.get(\"table_name\"),\n \"schema\": dataset_data.get(\"schema\"),\n \"columns\": updated_columns,\n \"owners\": [owner[\"id\"] for owner in dataset_data.get(\"owners\", [])],\n \"metrics\": updated_metrics,\n \"extra\": dataset_data.get(\"extra\"),\n \"description\": dataset_data.get(\"description\"),\n \"sql\": dataset_data.get(\"sql\"),\n \"cache_timeout\": dataset_data.get(\"cache_timeout\"),\n \"catalog\": dataset_data.get(\"catalog\"),\n \"default_endpoint\": dataset_data.get(\"default_endpoint\"),\n \"external_url\": dataset_data.get(\"external_url\"),\n \"fetch_values_predicate\": dataset_data.get(\"fetch_values_predicate\"),\n \"filter_select_enabled\": dataset_data.get(\"filter_select_enabled\"),\n \"is_managed_externally\": dataset_data.get(\"is_managed_externally\"),\n \"is_sqllab_view\": dataset_data.get(\"is_sqllab_view\"),\n \"main_dttm_col\": dataset_data.get(\"main_dttm_col\"),\n \"normalize_columns\": dataset_data.get(\"normalize_columns\"),\n \"offset\": dataset_data.get(\"offset\"),\n \"template_params\": dataset_data.get(\"template_params\"),\n }\n \n payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}\n\n superset_client.update_dataset(dataset_id, payload_for_update)\n app_logger.info(\"[run_mapping][Success] Dataset %d columns' verbose_name updated.\", dataset_id)\n else:\n app_logger.info(\"[run_mapping][State] No changes in columns' verbose_name, skipping update.\")\n\n except (AssertionError, FileNotFoundError, Exception) as e:\n app_logger.error(\"[run_mapping][Failure] %s\", e, exc_info=True)\n return\n # [/DEF:run_mapping:Function]\n# [/DEF:DatasetMapper:Class]\n" + "body": "# [DEF:DatasetMapper:Class]\n# @PURPOSE: Класс для меппинга и обновления verbose_map в датасетах Superset.\nclass DatasetMapper:\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the mapper.\n # @POST: Объект DatasetMapper инициализирован.\n def __init__(self):\n pass\n # [/DEF:__init__:Function]\n\n # [DEF:get_postgres_comments:Function]\n # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.\n # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).\n # @PRE: table_name и table_schema должны быть строками.\n # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.\n # @THROW: Exception - При ошибках подключения или выполнения запроса к БД.\n # @PARAM: db_config (Dict) - Конфигурация для подключения к БД.\n # @PARAM: table_name (str) - Имя таблицы.\n # @PARAM: table_schema (str) - Схема таблицы.\n # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.\n def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:\n with belief_scope(\"Fetch comments from PostgreSQL\"):\n log.reason(\"Fetching comments from PostgreSQL\", payload={\"table_schema\": table_schema, \"table_name\": table_name})\n query = f\"\"\"\n SELECT \n cols.column_name,\n CASE \n WHEN pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ) LIKE '%|%' THEN \n split_part(\n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ), \n '|', \n 1\n )\n ELSE \n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n )\n END AS column_comment\n FROM \n information_schema.columns cols \n WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';\n \"\"\"\n comments = {}\n try:\n with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:\n cursor.execute(query)\n for row in cursor.fetchall():\n if row[1]:\n comments[row[0]] = row[1]\n log.reflect(\"PostgreSQL comments fetched\", payload={\"count\": len(comments)})\n except Exception as e:\n log.explore(\"Failed to fetch PostgreSQL comments\", payload={\"table_schema\": table_schema, \"table_name\": table_name}, error=str(e))\n raise\n return comments\n # [/DEF:get_postgres_comments:Function]\n\n # [DEF:load_excel_mappings:Function]\n # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.\n # @PRE: file_path должен указывать на существующий XLSX файл.\n # @POST: Возвращается словарь с меппингами из файла.\n # @THROW: Exception - При ошибках чтения файла или парсинга.\n # @PARAM: file_path (str) - Путь к XLSX файлу.\n # @RETURN: Dict[str, str] - Словарь с меппингами.\n def load_excel_mappings(self, file_path: str) -> Dict[str, str]:\n with belief_scope(\"Load mappings from Excel\"):\n log.reason(\"Loading mappings from Excel\", payload={\"file_path\": file_path})\n try:\n df = pd.read_excel(file_path)\n mappings = df.set_index('column_name')['verbose_name'].to_dict()\n log.reflect(\"Excel mappings loaded\", payload={\"count\": len(mappings)})\n return mappings\n except Exception as e:\n log.explore(\"Failed to load Excel mappings\", payload={\"file_path\": file_path}, error=str(e))\n raise\n # [/DEF:load_excel_mappings:Function]\n\n # [DEF:run_mapping:Function]\n # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.\n # @PRE: superset_client должен быть авторизован.\n # @PRE: dataset_id должен быть существующим ID в Superset.\n # @POST: Если найдены изменения, датасет в Superset обновлен через API.\n # @RELATION: CALLS -> self.get_postgres_comments\n # @RELATION: CALLS -> self.load_excel_mappings\n # @RELATION: CALLS -> superset_client.get_dataset\n # @RELATION: CALLS -> superset_client.update_dataset\n # @PARAM: superset_client (Any) - Клиент Superset.\n # @PARAM: dataset_id (int) - ID датасета для обновления.\n # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').\n # @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.\n # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.\n # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.\n # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.\n def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):\n with belief_scope(f\"Run dataset mapping for ID {dataset_id}\"):\n log.reason(\"Starting dataset mapping\", payload={\"dataset_id\": dataset_id, \"source\": source})\n mappings: Dict[str, str] = {}\n \n try:\n if source in ['postgres', 'both']:\n if not (postgres_config and table_name and table_schema):\n raise ValueError(\"Postgres config is required.\")\n mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))\n if source in ['excel', 'both']:\n if not excel_path:\n raise ValueError(\"Excel path is required.\")\n mappings.update(self.load_excel_mappings(excel_path))\n if source not in ['postgres', 'excel', 'both']:\n log.explore(\"Invalid mapping source\", payload={\"source\": source}, error=f\"Invalid source: {source}\")\n return\n\n dataset_response = superset_client.get_dataset(dataset_id)\n dataset_data = dataset_response['result']\n \n original_columns = dataset_data.get('columns', [])\n updated_columns = []\n changes_made = False\n\n for column in original_columns:\n col_name = column.get('column_name')\n \n new_column = {\n \"column_name\": col_name,\n \"id\": column.get(\"id\"),\n \"advanced_data_type\": column.get(\"advanced_data_type\"),\n \"description\": column.get(\"description\"),\n \"expression\": column.get(\"expression\"),\n \"extra\": column.get(\"extra\"),\n \"filterable\": column.get(\"filterable\"),\n \"groupby\": column.get(\"groupby\"),\n \"is_active\": column.get(\"is_active\"),\n \"is_dttm\": column.get(\"is_dttm\"),\n \"python_date_format\": column.get(\"python_date_format\"),\n \"type\": column.get(\"type\"),\n \"uuid\": column.get(\"uuid\"),\n \"verbose_name\": column.get(\"verbose_name\"),\n }\n \n new_column = {k: v for k, v in new_column.items() if v is not None}\n\n if col_name in mappings:\n mapping_value = mappings[col_name]\n if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:\n new_column['verbose_name'] = mapping_value\n changes_made = True\n \n updated_columns.append(new_column)\n\n updated_metrics = []\n for metric in dataset_data.get(\"metrics\", []):\n new_metric = {\n \"id\": metric.get(\"id\"),\n \"metric_name\": metric.get(\"metric_name\"),\n \"expression\": metric.get(\"expression\"),\n \"verbose_name\": metric.get(\"verbose_name\"),\n \"description\": metric.get(\"description\"),\n \"d3format\": metric.get(\"d3format\"),\n \"currency\": metric.get(\"currency\"),\n \"extra\": metric.get(\"extra\"),\n \"warning_text\": metric.get(\"warning_text\"),\n \"metric_type\": metric.get(\"metric_type\"),\n \"uuid\": metric.get(\"uuid\"),\n }\n updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})\n\n if changes_made:\n payload_for_update = {\n \"database_id\": dataset_data.get(\"database\", {}).get(\"id\"),\n \"table_name\": dataset_data.get(\"table_name\"),\n \"schema\": dataset_data.get(\"schema\"),\n \"columns\": updated_columns,\n \"owners\": [owner[\"id\"] for owner in dataset_data.get(\"owners\", [])],\n \"metrics\": updated_metrics,\n \"extra\": dataset_data.get(\"extra\"),\n \"description\": dataset_data.get(\"description\"),\n \"sql\": dataset_data.get(\"sql\"),\n \"cache_timeout\": dataset_data.get(\"cache_timeout\"),\n \"catalog\": dataset_data.get(\"catalog\"),\n \"default_endpoint\": dataset_data.get(\"default_endpoint\"),\n \"external_url\": dataset_data.get(\"external_url\"),\n \"fetch_values_predicate\": dataset_data.get(\"fetch_values_predicate\"),\n \"filter_select_enabled\": dataset_data.get(\"filter_select_enabled\"),\n \"is_managed_externally\": dataset_data.get(\"is_managed_externally\"),\n \"is_sqllab_view\": dataset_data.get(\"is_sqllab_view\"),\n \"main_dttm_col\": dataset_data.get(\"main_dttm_col\"),\n \"normalize_columns\": dataset_data.get(\"normalize_columns\"),\n \"offset\": dataset_data.get(\"offset\"),\n \"template_params\": dataset_data.get(\"template_params\"),\n }\n \n payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}\n\n superset_client.update_dataset(dataset_id, payload_for_update)\n log.reflect(\"Dataset verbose_name updated\", payload={\"dataset_id\": dataset_id})\n else:\n log.reflect(\"No changes in verbose_name, skipping update\", payload={\"dataset_id\": dataset_id})\n\n except (FileNotFoundError, Exception) as e:\n log.explore(\"Dataset mapping failed\", payload={\"dataset_id\": dataset_id, \"source\": source}, error=str(e))\n return\n # [/DEF:run_mapping:Function]\n# [/DEF:DatasetMapper:Class]\n" }, { "contract_id": "get_postgres_comments", "contract_type": "Function", "file_path": "backend/src/core/utils/dataset_mapper.py", - "start_line": 28, - "end_line": 91, + "start_line": 31, + "end_line": 94, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48749,14 +49505,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_postgres_comments:Function]\n # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.\n # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).\n # @PRE: table_name и table_schema должны быть строками.\n # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.\n # @THROW: Exception - При ошибках подключения или выполнения запроса к БД.\n # @PARAM: db_config (Dict) - Конфигурация для подключения к БД.\n # @PARAM: table_name (str) - Имя таблицы.\n # @PARAM: table_schema (str) - Схема таблицы.\n # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.\n def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:\n with belief_scope(\"Fetch comments from PostgreSQL\"):\n app_logger.info(\"[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.\", table_schema, table_name)\n query = f\"\"\"\n SELECT \n cols.column_name,\n CASE \n WHEN pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ) LIKE '%|%' THEN \n split_part(\n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ), \n '|', \n 1\n )\n ELSE \n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n )\n END AS column_comment\n FROM \n information_schema.columns cols \n WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';\n \"\"\"\n comments = {}\n try:\n with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:\n cursor.execute(query)\n for row in cursor.fetchall():\n if row[1]:\n comments[row[0]] = row[1]\n app_logger.info(\"[get_postgres_comments][Success] Fetched %d comments.\", len(comments))\n except Exception as e:\n app_logger.error(\"[get_postgres_comments][Failure] %s\", e, exc_info=True)\n raise\n return comments\n # [/DEF:get_postgres_comments:Function]\n" + "body": " # [DEF:get_postgres_comments:Function]\n # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.\n # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).\n # @PRE: table_name и table_schema должны быть строками.\n # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.\n # @THROW: Exception - При ошибках подключения или выполнения запроса к БД.\n # @PARAM: db_config (Dict) - Конфигурация для подключения к БД.\n # @PARAM: table_name (str) - Имя таблицы.\n # @PARAM: table_schema (str) - Схема таблицы.\n # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.\n def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:\n with belief_scope(\"Fetch comments from PostgreSQL\"):\n log.reason(\"Fetching comments from PostgreSQL\", payload={\"table_schema\": table_schema, \"table_name\": table_name})\n query = f\"\"\"\n SELECT \n cols.column_name,\n CASE \n WHEN pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ) LIKE '%|%' THEN \n split_part(\n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n ), \n '|', \n 1\n )\n ELSE \n pg_catalog.col_description(\n (SELECT c.oid \n FROM pg_catalog.pg_class c \n JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \n WHERE c.relname = cols.table_name \n AND n.nspname = cols.table_schema),\n cols.ordinal_position::int\n )\n END AS column_comment\n FROM \n information_schema.columns cols \n WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';\n \"\"\"\n comments = {}\n try:\n with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:\n cursor.execute(query)\n for row in cursor.fetchall():\n if row[1]:\n comments[row[0]] = row[1]\n log.reflect(\"PostgreSQL comments fetched\", payload={\"count\": len(comments)})\n except Exception as e:\n log.explore(\"Failed to fetch PostgreSQL comments\", payload={\"table_schema\": table_schema, \"table_name\": table_name}, error=str(e))\n raise\n return comments\n # [/DEF:get_postgres_comments:Function]\n" }, { "contract_id": "load_excel_mappings", "contract_type": "Function", "file_path": "backend/src/core/utils/dataset_mapper.py", - "start_line": 93, - "end_line": 111, + "start_line": 96, + "end_line": 114, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48807,14 +49563,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:load_excel_mappings:Function]\n # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.\n # @PRE: file_path должен указывать на существующий XLSX файл.\n # @POST: Возвращается словарь с меппингами из файла.\n # @THROW: Exception - При ошибках чтения файла или парсинга.\n # @PARAM: file_path (str) - Путь к XLSX файлу.\n # @RETURN: Dict[str, str] - Словарь с меппингами.\n def load_excel_mappings(self, file_path: str) -> Dict[str, str]:\n with belief_scope(\"Load mappings from Excel\"):\n app_logger.info(\"[load_excel_mappings][Enter] Loading mappings from %s.\", file_path)\n try:\n df = pd.read_excel(file_path)\n mappings = df.set_index('column_name')['verbose_name'].to_dict()\n app_logger.info(\"[load_excel_mappings][Success] Loaded %d mappings.\", len(mappings))\n return mappings\n except Exception as e:\n app_logger.error(\"[load_excel_mappings][Failure] %s\", e, exc_info=True)\n raise\n # [/DEF:load_excel_mappings:Function]\n" + "body": " # [DEF:load_excel_mappings:Function]\n # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.\n # @PRE: file_path должен указывать на существующий XLSX файл.\n # @POST: Возвращается словарь с меппингами из файла.\n # @THROW: Exception - При ошибках чтения файла или парсинга.\n # @PARAM: file_path (str) - Путь к XLSX файлу.\n # @RETURN: Dict[str, str] - Словарь с меппингами.\n def load_excel_mappings(self, file_path: str) -> Dict[str, str]:\n with belief_scope(\"Load mappings from Excel\"):\n log.reason(\"Loading mappings from Excel\", payload={\"file_path\": file_path})\n try:\n df = pd.read_excel(file_path)\n mappings = df.set_index('column_name')['verbose_name'].to_dict()\n log.reflect(\"Excel mappings loaded\", payload={\"count\": len(mappings)})\n return mappings\n except Exception as e:\n log.explore(\"Failed to load Excel mappings\", payload={\"file_path\": file_path}, error=str(e))\n raise\n # [/DEF:load_excel_mappings:Function]\n" }, { "contract_id": "run_mapping", "contract_type": "Function", "file_path": "backend/src/core/utils/dataset_mapper.py", - "start_line": 113, - "end_line": 234, + "start_line": 116, + "end_line": 239, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48885,14 +49641,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:run_mapping:Function]\n # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.\n # @PRE: superset_client должен быть авторизован.\n # @PRE: dataset_id должен быть существующим ID в Superset.\n # @POST: Если найдены изменения, датасет в Superset обновлен через API.\n # @RELATION: CALLS -> self.get_postgres_comments\n # @RELATION: CALLS -> self.load_excel_mappings\n # @RELATION: CALLS -> superset_client.get_dataset\n # @RELATION: CALLS -> superset_client.update_dataset\n # @PARAM: superset_client (Any) - Клиент Superset.\n # @PARAM: dataset_id (int) - ID датасета для обновления.\n # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').\n # @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.\n # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.\n # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.\n # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.\n def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):\n with belief_scope(f\"Run dataset mapping for ID {dataset_id}\"):\n app_logger.info(\"[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.\", dataset_id, source)\n mappings: Dict[str, str] = {}\n \n try:\n if source in ['postgres', 'both']:\n assert postgres_config and table_name and table_schema, \"Postgres config is required.\"\n mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))\n if source in ['excel', 'both']:\n assert excel_path, \"Excel path is required.\"\n mappings.update(self.load_excel_mappings(excel_path))\n if source not in ['postgres', 'excel', 'both']:\n app_logger.error(\"[run_mapping][Failure] Invalid source: %s.\", source)\n return\n\n dataset_response = superset_client.get_dataset(dataset_id)\n dataset_data = dataset_response['result']\n \n original_columns = dataset_data.get('columns', [])\n updated_columns = []\n changes_made = False\n\n for column in original_columns:\n col_name = column.get('column_name')\n \n new_column = {\n \"column_name\": col_name,\n \"id\": column.get(\"id\"),\n \"advanced_data_type\": column.get(\"advanced_data_type\"),\n \"description\": column.get(\"description\"),\n \"expression\": column.get(\"expression\"),\n \"extra\": column.get(\"extra\"),\n \"filterable\": column.get(\"filterable\"),\n \"groupby\": column.get(\"groupby\"),\n \"is_active\": column.get(\"is_active\"),\n \"is_dttm\": column.get(\"is_dttm\"),\n \"python_date_format\": column.get(\"python_date_format\"),\n \"type\": column.get(\"type\"),\n \"uuid\": column.get(\"uuid\"),\n \"verbose_name\": column.get(\"verbose_name\"),\n }\n \n new_column = {k: v for k, v in new_column.items() if v is not None}\n\n if col_name in mappings:\n mapping_value = mappings[col_name]\n if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:\n new_column['verbose_name'] = mapping_value\n changes_made = True\n \n updated_columns.append(new_column)\n\n updated_metrics = []\n for metric in dataset_data.get(\"metrics\", []):\n new_metric = {\n \"id\": metric.get(\"id\"),\n \"metric_name\": metric.get(\"metric_name\"),\n \"expression\": metric.get(\"expression\"),\n \"verbose_name\": metric.get(\"verbose_name\"),\n \"description\": metric.get(\"description\"),\n \"d3format\": metric.get(\"d3format\"),\n \"currency\": metric.get(\"currency\"),\n \"extra\": metric.get(\"extra\"),\n \"warning_text\": metric.get(\"warning_text\"),\n \"metric_type\": metric.get(\"metric_type\"),\n \"uuid\": metric.get(\"uuid\"),\n }\n updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})\n\n if changes_made:\n payload_for_update = {\n \"database_id\": dataset_data.get(\"database\", {}).get(\"id\"),\n \"table_name\": dataset_data.get(\"table_name\"),\n \"schema\": dataset_data.get(\"schema\"),\n \"columns\": updated_columns,\n \"owners\": [owner[\"id\"] for owner in dataset_data.get(\"owners\", [])],\n \"metrics\": updated_metrics,\n \"extra\": dataset_data.get(\"extra\"),\n \"description\": dataset_data.get(\"description\"),\n \"sql\": dataset_data.get(\"sql\"),\n \"cache_timeout\": dataset_data.get(\"cache_timeout\"),\n \"catalog\": dataset_data.get(\"catalog\"),\n \"default_endpoint\": dataset_data.get(\"default_endpoint\"),\n \"external_url\": dataset_data.get(\"external_url\"),\n \"fetch_values_predicate\": dataset_data.get(\"fetch_values_predicate\"),\n \"filter_select_enabled\": dataset_data.get(\"filter_select_enabled\"),\n \"is_managed_externally\": dataset_data.get(\"is_managed_externally\"),\n \"is_sqllab_view\": dataset_data.get(\"is_sqllab_view\"),\n \"main_dttm_col\": dataset_data.get(\"main_dttm_col\"),\n \"normalize_columns\": dataset_data.get(\"normalize_columns\"),\n \"offset\": dataset_data.get(\"offset\"),\n \"template_params\": dataset_data.get(\"template_params\"),\n }\n \n payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}\n\n superset_client.update_dataset(dataset_id, payload_for_update)\n app_logger.info(\"[run_mapping][Success] Dataset %d columns' verbose_name updated.\", dataset_id)\n else:\n app_logger.info(\"[run_mapping][State] No changes in columns' verbose_name, skipping update.\")\n\n except (AssertionError, FileNotFoundError, Exception) as e:\n app_logger.error(\"[run_mapping][Failure] %s\", e, exc_info=True)\n return\n # [/DEF:run_mapping:Function]\n" + "body": " # [DEF:run_mapping:Function]\n # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.\n # @PRE: superset_client должен быть авторизован.\n # @PRE: dataset_id должен быть существующим ID в Superset.\n # @POST: Если найдены изменения, датасет в Superset обновлен через API.\n # @RELATION: CALLS -> self.get_postgres_comments\n # @RELATION: CALLS -> self.load_excel_mappings\n # @RELATION: CALLS -> superset_client.get_dataset\n # @RELATION: CALLS -> superset_client.update_dataset\n # @PARAM: superset_client (Any) - Клиент Superset.\n # @PARAM: dataset_id (int) - ID датасета для обновления.\n # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').\n # @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.\n # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.\n # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.\n # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.\n def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):\n with belief_scope(f\"Run dataset mapping for ID {dataset_id}\"):\n log.reason(\"Starting dataset mapping\", payload={\"dataset_id\": dataset_id, \"source\": source})\n mappings: Dict[str, str] = {}\n \n try:\n if source in ['postgres', 'both']:\n if not (postgres_config and table_name and table_schema):\n raise ValueError(\"Postgres config is required.\")\n mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))\n if source in ['excel', 'both']:\n if not excel_path:\n raise ValueError(\"Excel path is required.\")\n mappings.update(self.load_excel_mappings(excel_path))\n if source not in ['postgres', 'excel', 'both']:\n log.explore(\"Invalid mapping source\", payload={\"source\": source}, error=f\"Invalid source: {source}\")\n return\n\n dataset_response = superset_client.get_dataset(dataset_id)\n dataset_data = dataset_response['result']\n \n original_columns = dataset_data.get('columns', [])\n updated_columns = []\n changes_made = False\n\n for column in original_columns:\n col_name = column.get('column_name')\n \n new_column = {\n \"column_name\": col_name,\n \"id\": column.get(\"id\"),\n \"advanced_data_type\": column.get(\"advanced_data_type\"),\n \"description\": column.get(\"description\"),\n \"expression\": column.get(\"expression\"),\n \"extra\": column.get(\"extra\"),\n \"filterable\": column.get(\"filterable\"),\n \"groupby\": column.get(\"groupby\"),\n \"is_active\": column.get(\"is_active\"),\n \"is_dttm\": column.get(\"is_dttm\"),\n \"python_date_format\": column.get(\"python_date_format\"),\n \"type\": column.get(\"type\"),\n \"uuid\": column.get(\"uuid\"),\n \"verbose_name\": column.get(\"verbose_name\"),\n }\n \n new_column = {k: v for k, v in new_column.items() if v is not None}\n\n if col_name in mappings:\n mapping_value = mappings[col_name]\n if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value:\n new_column['verbose_name'] = mapping_value\n changes_made = True\n \n updated_columns.append(new_column)\n\n updated_metrics = []\n for metric in dataset_data.get(\"metrics\", []):\n new_metric = {\n \"id\": metric.get(\"id\"),\n \"metric_name\": metric.get(\"metric_name\"),\n \"expression\": metric.get(\"expression\"),\n \"verbose_name\": metric.get(\"verbose_name\"),\n \"description\": metric.get(\"description\"),\n \"d3format\": metric.get(\"d3format\"),\n \"currency\": metric.get(\"currency\"),\n \"extra\": metric.get(\"extra\"),\n \"warning_text\": metric.get(\"warning_text\"),\n \"metric_type\": metric.get(\"metric_type\"),\n \"uuid\": metric.get(\"uuid\"),\n }\n updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})\n\n if changes_made:\n payload_for_update = {\n \"database_id\": dataset_data.get(\"database\", {}).get(\"id\"),\n \"table_name\": dataset_data.get(\"table_name\"),\n \"schema\": dataset_data.get(\"schema\"),\n \"columns\": updated_columns,\n \"owners\": [owner[\"id\"] for owner in dataset_data.get(\"owners\", [])],\n \"metrics\": updated_metrics,\n \"extra\": dataset_data.get(\"extra\"),\n \"description\": dataset_data.get(\"description\"),\n \"sql\": dataset_data.get(\"sql\"),\n \"cache_timeout\": dataset_data.get(\"cache_timeout\"),\n \"catalog\": dataset_data.get(\"catalog\"),\n \"default_endpoint\": dataset_data.get(\"default_endpoint\"),\n \"external_url\": dataset_data.get(\"external_url\"),\n \"fetch_values_predicate\": dataset_data.get(\"fetch_values_predicate\"),\n \"filter_select_enabled\": dataset_data.get(\"filter_select_enabled\"),\n \"is_managed_externally\": dataset_data.get(\"is_managed_externally\"),\n \"is_sqllab_view\": dataset_data.get(\"is_sqllab_view\"),\n \"main_dttm_col\": dataset_data.get(\"main_dttm_col\"),\n \"normalize_columns\": dataset_data.get(\"normalize_columns\"),\n \"offset\": dataset_data.get(\"offset\"),\n \"template_params\": dataset_data.get(\"template_params\"),\n }\n \n payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}\n\n superset_client.update_dataset(dataset_id, payload_for_update)\n log.reflect(\"Dataset verbose_name updated\", payload={\"dataset_id\": dataset_id})\n else:\n log.reflect(\"No changes in verbose_name, skipping update\", payload={\"dataset_id\": dataset_id})\n\n except (FileNotFoundError, Exception) as e:\n log.explore(\"Dataset mapping failed\", payload={\"dataset_id\": dataset_id, \"source\": source}, error=str(e))\n return\n # [/DEF:run_mapping:Function]\n" }, { "contract_id": "FileIO", "contract_type": "Module", "file_path": "backend/src/core/utils/fileio.py", "start_line": 1, - "end_line": 487, + "end_line": 489, "tier": "STANDARD", "complexity": 1, "metadata": { @@ -48936,14 +49692,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:FileIO:Module]\n#\n# @TIER: STANDARD\n# @SEMANTICS: file, io, zip, yaml, temp, archive, utility\n# @PURPOSE: Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.\n# @LAYER: Infra\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders\n\n# [SECTION: IMPORTS]\nimport os\nimport re\nimport zipfile\nfrom pathlib import Path\nfrom typing import Any, Optional, Tuple, Dict, List, Union, LiteralString, Generator\nfrom contextlib import contextmanager\nimport tempfile\nfrom datetime import date, datetime\nimport shutil\nimport zlib\nfrom dataclasses import dataclass\nfrom ..logger import logger as app_logger, belief_scope\n# [/SECTION]\n\n# [DEF:InvalidZipFormatError:Class]\n# @PURPOSE: Exception raised when a file is not a valid ZIP archive.\nclass InvalidZipFormatError(Exception):\n pass\n# [/DEF:InvalidZipFormatError:Class]\n\n# [DEF:create_temp_file:Function]\n# @PURPOSE: Контекстный менеджер для создания временного файла или директории с гарантированным удалением.\n# @PRE: suffix должен быть строкой, определяющей тип ресурса.\n# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.\n# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл.\n# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория.\n# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb').\n# @YIELDS: Path - Путь к временному ресурсу.\n# @THROW: IOError - При ошибках создания ресурса.\n@contextmanager\ndef create_temp_file(content: Optional[bytes] = None, suffix: str = \".zip\", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:\n with belief_scope(\"Create temporary resource\"):\n resource_path = None\n is_dir = suffix.startswith('.dir')\n try:\n if is_dir:\n with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir:\n resource_path = Path(temp_dir)\n app_logger.debug(\"[create_temp_file][State] Created temporary directory: %s\", resource_path)\n yield resource_path\n else:\n fd, temp_path_str = tempfile.mkstemp(suffix=suffix)\n resource_path = Path(temp_path_str)\n os.close(fd)\n if content:\n resource_path.write_bytes(content)\n app_logger.debug(\"[create_temp_file][State] Created temporary file: %s\", resource_path)\n yield resource_path\n finally:\n if resource_path and resource_path.exists() and not dry_run:\n try:\n if resource_path.is_dir():\n shutil.rmtree(resource_path)\n app_logger.debug(\"[create_temp_file][Cleanup] Removed temporary directory: %s\", resource_path)\n else:\n resource_path.unlink()\n app_logger.debug(\"[create_temp_file][Cleanup] Removed temporary file: %s\", resource_path)\n except OSError as e:\n app_logger.error(\"[create_temp_file][Failure] Error during cleanup of %s: %s\", resource_path, e)\n# [/DEF:create_temp_file:Function]\n\n# [DEF:remove_empty_directories:Function]\n# @PURPOSE: Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.\n# @PRE: root_dir должен быть путем к существующей директории.\n# @POST: Все пустые поддиректории удалены, возвращено их количество.\n# @PARAM: root_dir (str) - Путь к корневой директории для очистки.\n# @RETURN: int - Количество удаленных директорий.\ndef remove_empty_directories(root_dir: str) -> int:\n with belief_scope(f\"Remove empty directories in {root_dir}\"):\n app_logger.info(\"[remove_empty_directories][Enter] Starting cleanup of empty directories in %s\", root_dir)\n removed_count = 0\n if not os.path.isdir(root_dir):\n app_logger.error(\"[remove_empty_directories][Failure] Directory not found: %s\", root_dir)\n return 0\n for current_dir, _, _ in os.walk(root_dir, topdown=False):\n if not os.listdir(current_dir):\n try:\n os.rmdir(current_dir)\n removed_count += 1\n app_logger.info(\"[remove_empty_directories][State] Removed empty directory: %s\", current_dir)\n except OSError as e:\n app_logger.error(\"[remove_empty_directories][Failure] Failed to remove %s: %s\", current_dir, e)\n app_logger.info(\"[remove_empty_directories][Exit] Removed %d empty directories.\", removed_count)\n return removed_count\n# [/DEF:remove_empty_directories:Function]\n\n# [DEF:read_dashboard_from_disk:Function]\n# @PURPOSE: Читает бинарное содержимое файла с диска.\n# @PRE: file_path должен указывать на существующий файл.\n# @POST: Возвращает байты содержимого и имя файла.\n# @PARAM: file_path (str) - Путь к файлу.\n# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла).\n# @THROW: FileNotFoundError - Если файл не найден.\ndef read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:\n with belief_scope(f\"Read dashboard from {file_path}\"):\n path = Path(file_path)\n assert path.is_file(), f\"Файл дашборда не найден: {file_path}\"\n app_logger.info(\"[read_dashboard_from_disk][Enter] Reading file: %s\", file_path)\n content = path.read_bytes()\n if not content:\n app_logger.warning(\"[read_dashboard_from_disk][Warning] File is empty: %s\", file_path)\n return content, path.name\n# [/DEF:read_dashboard_from_disk:Function]\n\n# [DEF:calculate_crc32:Function]\n# @PURPOSE: Вычисляет контрольную сумму CRC32 для файла.\n# @PRE: file_path должен быть объектом Path к существующему файлу.\n# @POST: Возвращает 8-значную hex-строку CRC32.\n# @PARAM: file_path (Path) - Путь к файлу.\n# @RETURN: str - 8-значное шестнадцатеричное представление CRC32.\n# @THROW: IOError - При ошибках чтения файла.\ndef calculate_crc32(file_path: Path) -> str:\n with belief_scope(f\"Calculate CRC32 for {file_path}\"):\n with open(file_path, 'rb') as f:\n crc32_value = zlib.crc32(f.read())\n return f\"{crc32_value:08x}\"\n# [/DEF:calculate_crc32:Function]\n\n# [SECTION: DATA_CLASSES]\n# [DEF:RetentionPolicy:DataClass]\n# @PURPOSE: Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).\n@dataclass\nclass RetentionPolicy:\n daily: int = 7\n weekly: int = 4\n monthly: int = 12\n# [/DEF:RetentionPolicy:DataClass]\n# [/SECTION]\n\n# [DEF:archive_exports:Function]\n# @PURPOSE: Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.\n# @PRE: output_dir должен быть путем к существующей директории.\n# @POST: Старые или дублирующиеся архивы удалены согласно политике.\n# @RELATION: CALLS -> apply_retention_policy\n# @RELATION: CALLS -> calculate_crc32\n# @PARAM: output_dir (str) - Директория с архивами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32.\ndef archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:\n with belief_scope(f\"Archive exports in {output_dir}\"):\n output_path = Path(output_dir)\n if not output_path.is_dir():\n app_logger.warning(\"[archive_exports][Skip] Archive directory not found: %s\", output_dir)\n return\n\n app_logger.info(\"[archive_exports][Enter] Managing archive in %s\", output_dir)\n \n # 1. Collect all zip files\n zip_files = list(output_path.glob(\"*.zip\"))\n if not zip_files:\n app_logger.info(\"[archive_exports][State] No zip files found in %s\", output_dir)\n return\n\n # 2. Deduplication\n if deduplicate:\n app_logger.info(\"[archive_exports][State] Starting deduplication...\")\n checksums = {}\n files_to_remove = []\n \n # Sort by modification time (newest first) to keep the latest version\n zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)\n \n for file_path in zip_files:\n try:\n crc = calculate_crc32(file_path)\n if crc in checksums:\n files_to_remove.append(file_path)\n app_logger.debug(\"[archive_exports][State] Duplicate found: %s (same as %s)\", file_path.name, checksums[crc].name)\n else:\n checksums[crc] = file_path\n except Exception as e:\n app_logger.error(\"[archive_exports][Failure] Failed to calculate CRC32 for %s: %s\", file_path, e)\n \n for f in files_to_remove:\n try:\n f.unlink()\n zip_files.remove(f)\n app_logger.info(\"[archive_exports][State] Removed duplicate: %s\", f.name)\n except OSError as e:\n app_logger.error(\"[archive_exports][Failure] Failed to remove duplicate %s: %s\", f, e)\n\n # 3. Retention Policy\n files_with_dates = []\n for file_path in zip_files:\n # Try to extract date from filename\n # Pattern: ..._YYYYMMDD_HHMMSS.zip or ..._YYYYMMDD.zip\n match = re.search(r'_(\\d{8})_', file_path.name)\n file_date = None\n if match:\n try:\n date_str = match.group(1)\n file_date = datetime.strptime(date_str, \"%Y%m%d\").date()\n except ValueError:\n pass\n \n if not file_date:\n # Fallback to modification time\n file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date()\n \n files_with_dates.append((file_path, file_date))\n\n files_to_keep = apply_retention_policy(files_with_dates, policy)\n \n for file_path, _ in files_with_dates:\n if file_path not in files_to_keep:\n try:\n file_path.unlink()\n app_logger.info(\"[archive_exports][State] Removed by retention policy: %s\", file_path.name)\n except OSError as e:\n app_logger.error(\"[archive_exports][Failure] Failed to remove %s: %s\", file_path, e)\n# [/DEF:archive_exports:Function]\n\n# [DEF:apply_retention_policy:Function]\n# @PURPOSE: (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.\n# @PRE: files_with_dates is a list of (Path, date) tuples.\n# @POST: Returns a set of files to keep.\n# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @RETURN: set - Множество путей к файлам, которые должны быть сохранены.\ndef apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:\n with belief_scope(\"Apply retention policy\"):\n # Сортируем по дате (от новой к старой)\n sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True) \n # Словарь для хранения файлов по категориям \n daily_files = [] \n weekly_files = [] \n monthly_files = [] \n today = date.today() \n for file_path, file_date in sorted_files: \n # Ежедневные \n if (today - file_date).days < policy.daily: \n daily_files.append(file_path) \n # Еженедельные \n elif (today - file_date).days < policy.weekly * 7: \n weekly_files.append(file_path) \n # Ежемесячные \n elif (today - file_date).days < policy.monthly * 30: \n monthly_files.append(file_path) \n # Возвращаем множество файлов, которые нужно сохранить \n files_to_keep = set() \n files_to_keep.update(daily_files) \n files_to_keep.update(weekly_files[:policy.weekly]) \n files_to_keep.update(monthly_files[:policy.monthly]) \n app_logger.debug(\"[apply_retention_policy][State] Keeping %d files according to retention policy\", len(files_to_keep))\n return files_to_keep\n# [/DEF:apply_retention_policy:Function]\n\n# [DEF:save_and_unpack_dashboard:Function]\n# @PURPOSE: Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.\n# @PRE: zip_content должен быть байтами валидного ZIP-архива.\n# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.\n# @PARAM: zip_content (bytes) - Содержимое ZIP-архива.\n# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения.\n# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив.\n# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения.\n# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой.\n# @THROW: InvalidZipFormatError - При ошибке формата ZIP.\ndef save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:\n with belief_scope(\"Save and unpack dashboard\"):\n app_logger.info(\"[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s\", unpack)\n try:\n output_path = Path(output_dir)\n output_path.mkdir(parents=True, exist_ok=True)\n zip_name = sanitize_filename(original_filename) if original_filename else f\"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip\"\n zip_path = output_path / zip_name\n zip_path.write_bytes(zip_content)\n app_logger.info(\"[save_and_unpack_dashboard][State] Dashboard saved to: %s\", zip_path)\n if unpack:\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(output_path)\n app_logger.info(\"[save_and_unpack_dashboard][State] Dashboard unpacked to: %s\", output_path)\n return zip_path, output_path\n return zip_path, None\n except zipfile.BadZipFile as e:\n app_logger.error(\"[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s\", e)\n raise InvalidZipFormatError(f\"Invalid ZIP file: {e}\") from e\n# [/DEF:save_and_unpack_dashboard:Function]\n\n# [DEF:update_yamls:Function]\n# @PURPOSE: Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.\n# @PRE: path должен быть существующей директорией.\n# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.\n# @RELATION: CALLS -> _update_yaml_file\n# @THROW: FileNotFoundError - Если `path` не существует.\n# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены.\n# @PARAM: path (str) - Путь к директории с YAML файлами.\n# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска.\n# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены.\ndef update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = \"dashboards\", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:\n with belief_scope(\"Update YAML configurations\"):\n app_logger.info(\"[update_yamls][Enter] Starting YAML configuration update.\")\n dir_path = Path(path)\n assert dir_path.is_dir(), f\"Путь {path} не существует или не является директорией\"\n \n configs: List[Dict[str, Any]] = db_configs or []\n \n for file_path in dir_path.rglob(\"*.yaml\"):\n _update_yaml_file(file_path, configs, regexp_pattern, replace_string)\n# [/DEF:update_yamls:Function]\n\n# [DEF:_update_yaml_file:Function]\n# @PURPOSE: (Helper) Обновляет один YAML файл.\n# @PRE: file_path должен быть объектом Path к существующему YAML файлу.\n# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.\n# @PARAM: file_path (Path) - Путь к файлу.\n# @PARAM: db_configs (List[Dict]) - Конфигурации.\n# @PARAM: regexp_pattern (Optional[str]) - Паттерн.\n# @PARAM: replace_string (Optional[str]) - Замена.\ndef _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:\n with belief_scope(f\"Update YAML file: {file_path}\"):\n # Читаем содержимое файла\n try:\n with open(file_path, 'r', encoding='utf-8') as f:\n content = f.read()\n except Exception as e:\n app_logger.error(\"[_update_yaml_file][Failure] Failed to read %s: %s\", file_path, e)\n return\n # Если задан pattern и replace_string, применяем замену по регулярному выражению\n if regexp_pattern and replace_string:\n try:\n new_content = re.sub(regexp_pattern, replace_string, content)\n if new_content != content:\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(new_content)\n app_logger.info(\"[_update_yaml_file][State] Updated %s using regex pattern\", file_path)\n except Exception as e:\n app_logger.error(\"[_update_yaml_file][Failure] Error applying regex to %s: %s\", file_path, e)\n # Если заданы конфигурации, заменяем значения (поддержка old/new)\n if db_configs:\n try:\n # Прямой текстовый заменитель для старых/новых значений, чтобы сохранить структуру файла\n modified_content = content\n for cfg in db_configs:\n # Ожидаем структуру: {'old': {...}, 'new': {...}}\n old_cfg = cfg.get('old', {})\n new_cfg = cfg.get('new', {})\n for key, old_val in old_cfg.items():\n if key in new_cfg:\n new_val = new_cfg[key]\n # Заменяем только точные совпадения старого значения в тексте YAML, используя ключ для контекста\n if isinstance(old_val, str):\n # Ищем паттерн: key: \"value\" или key: value\n key_pattern = re.escape(key)\n val_pattern = re.escape(old_val)\n # Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)\n pattern = rf'({key_pattern}\\s*:\\s*)([\"\\']?)({val_pattern})([\"\\']?)'\n \n # [DEF:replacer:Function]\n # @PURPOSE: Функция замены, сохраняющая кавычки если они были.\n # @PRE: match должен быть объектом совпадения регулярного выражения.\n # @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.\n def replacer(match):\n prefix = match.group(1)\n quote_open = match.group(2)\n quote_close = match.group(4)\n return f\"{prefix}{quote_open}{new_val}{quote_close}\"\n # [/DEF:replacer:Function]\n\n modified_content = re.sub(pattern, replacer, modified_content)\n app_logger.info(\"[_update_yaml_file][State] Replaced '%s' with '%s' for key %s in %s\", old_val, new_val, key, file_path)\n # Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(modified_content)\n except Exception as e:\n app_logger.error(\"[_update_yaml_file][Failure] Error performing raw replacement in %s: %s\", file_path, e)\n# [/DEF:_update_yaml_file:Function]\n\n# [DEF:create_dashboard_export:Function]\n# @PURPOSE: Создает ZIP-архив из указанных исходных путей.\n# @PRE: source_paths должен содержать существующие пути.\n# @POST: ZIP-архив создан по пути zip_path.\n# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива.\n# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации.\n# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения.\n# @RETURN: bool - `True` при успехе, `False` при ошибке.\ndef create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:\n with belief_scope(f\"Create dashboard export: {zip_path}\"):\n app_logger.info(\"[create_dashboard_export][Enter] Packing dashboard: %s -> %s\", source_paths, zip_path)\n try:\n exclude_ext = [ext.lower() for ext in exclude_extensions or []]\n with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n for src_path_str in source_paths:\n src_path = Path(src_path_str)\n assert src_path.exists(), f\"Путь не найден: {src_path}\"\n for item in src_path.rglob('*'):\n if item.is_file() and item.suffix.lower() not in exclude_ext:\n arcname = item.relative_to(src_path.parent)\n zipf.write(item, arcname)\n app_logger.info(\"[create_dashboard_export][Exit] Archive created: %s\", zip_path)\n return True\n except (IOError, zipfile.BadZipFile, AssertionError) as e:\n app_logger.error(\"[create_dashboard_export][Failure] Error: %s\", e, exc_info=True)\n return False\n# [/DEF:create_dashboard_export:Function]\n\n# [DEF:sanitize_filename:Function]\n# @PURPOSE: Очищает строку от символов, недопустимых в именах файлов.\n# @PRE: filename должен быть строкой.\n# @POST: Возвращает строку без спецсимволов.\n# @PARAM: filename (str) - Исходное имя файла.\n# @RETURN: str - Очищенная строка.\ndef sanitize_filename(filename: str) -> str:\n with belief_scope(f\"Sanitize filename: {filename}\"):\n return re.sub(r'[\\\\/*?:\"<>|]', \"_\", filename).strip()\n# [/DEF:sanitize_filename:Function]\n\n# [DEF:get_filename_from_headers:Function]\n# @PURPOSE: Извлекает имя файла из HTTP заголовка 'Content-Disposition'.\n# @PRE: headers должен быть словарем заголовков.\n# @POST: Возвращает имя файла или None, если заголовок отсутствует.\n# @PARAM: headers (dict) - Словарь HTTP заголовков.\n# @RETURN: Optional[str] - Имя файла or `None`.\ndef get_filename_from_headers(headers: dict) -> Optional[str]:\n with belief_scope(\"Get filename from headers\"):\n content_disposition = headers.get(\"Content-Disposition\", \"\")\n if match := re.search(r'filename=\"?([^\"]+)\"?', content_disposition):\n return match.group(1).strip()\n return None\n# [/DEF:get_filename_from_headers:Function]\n\n# [DEF:consolidate_archive_folders:Function]\n# @PURPOSE: Консолидирует директории архивов на основе общего слага в имени.\n# @PRE: root_directory должен быть объектом Path к существующей директории.\n# @POST: Директории с одинаковым префиксом объединены в одну.\n# @THROW: TypeError, ValueError - Если `root_directory` невалиден.\n# @PARAM: root_directory (Path) - Корневая директория для консолидации.\ndef consolidate_archive_folders(root_directory: Path) -> None:\n with belief_scope(f\"Consolidate archives in {root_directory}\"):\n assert isinstance(root_directory, Path), \"root_directory must be a Path object.\" \n assert root_directory.is_dir(), \"root_directory must be an existing directory.\" \n \n app_logger.info(\"[consolidate_archive_folders][Enter] Consolidating archives in %s\", root_directory) \n # Собираем все директории с архивами \n archive_dirs = [] \n for item in root_directory.iterdir(): \n if item.is_dir(): \n # Проверяем, есть ли в директории ZIP-архивы \n if any(item.glob(\"*.zip\")): \n archive_dirs.append(item) \n # Группируем по слагу (части имени до первого '_') \n slug_groups = {} \n for dir_path in archive_dirs: \n dir_name = dir_path.name \n slug = dir_name.split('_')[0] if '_' in dir_name else dir_name \n if slug not in slug_groups: \n slug_groups[slug] = [] \n slug_groups[slug].append(dir_path) \n # Для каждой группы консолидируем \n for slug, dirs in slug_groups.items(): \n if len(dirs) <= 1: \n continue \n # Создаем целевую директорию \n target_dir = root_directory / slug \n target_dir.mkdir(exist_ok=True) \n app_logger.info(\"[consolidate_archive_folders][State] Consolidating %d directories under %s\", len(dirs), target_dir) \n # Перемещаем содержимое \n for source_dir in dirs: \n if source_dir == target_dir: \n continue \n for item in source_dir.iterdir(): \n dest_item = target_dir / item.name \n try: \n if item.is_dir(): \n shutil.move(str(item), str(dest_item)) \n else: \n shutil.move(str(item), str(dest_item)) \n except Exception as e: \n app_logger.error(\"[consolidate_archive_folders][Failure] Failed to move %s to %s: %s\", item, dest_item, e) \n # Удаляем исходную директорию \n try: \n source_dir.rmdir() \n app_logger.info(\"[consolidate_archive_folders][State] Removed source directory: %s\", source_dir) \n except Exception as e: \n app_logger.error(\"[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s\", source_dir, e) \n# [/DEF:consolidate_archive_folders:Function]\n\n# [/DEF:FileIO:Module]\n" + "body": "# [DEF:FileIO:Module]\n#\n# @TIER: STANDARD\n# @SEMANTICS: file, io, zip, yaml, temp, archive, utility\n# @PURPOSE: Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.\n# @LAYER: Infra\n# @RELATION: DEPENDS_ON -> [LoggerModule]\n# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders\n\n# [SECTION: IMPORTS]\nimport os\nimport re\nimport zipfile\nfrom pathlib import Path\nfrom typing import Any, Optional, Tuple, Dict, List, Union, LiteralString, Generator\nfrom contextlib import contextmanager\nimport tempfile\nfrom datetime import date, datetime\nimport shutil\nimport zlib\nfrom dataclasses import dataclass\nfrom ..logger import logger as app_logger, belief_scope\nfrom ..cot_logger import MarkerLogger\n# [/SECTION]\n\nlog = MarkerLogger(\"FileIO\")\n\n# [DEF:InvalidZipFormatError:Class]\n# @PURPOSE: Exception raised when a file is not a valid ZIP archive.\nclass InvalidZipFormatError(Exception):\n pass\n# [/DEF:InvalidZipFormatError:Class]\n\n# [DEF:create_temp_file:Function]\n# @PURPOSE: Контекстный менеджер для создания временного файла или директории с гарантированным удалением.\n# @PRE: suffix должен быть строкой, определяющей тип ресурса.\n# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.\n# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл.\n# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория.\n# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb').\n# @YIELDS: Path - Путь к временному ресурсу.\n# @THROW: IOError - При ошибках создания ресурса.\n@contextmanager\ndef create_temp_file(content: Optional[bytes] = None, suffix: str = \".zip\", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:\n with belief_scope(\"Create temporary resource\"):\n resource_path = None\n is_dir = suffix.startswith('.dir')\n try:\n if is_dir:\n with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir:\n resource_path = Path(temp_dir)\n log.reason(\"Created temporary directory\", payload={\"path\": str(resource_path)})\n yield resource_path\n else:\n fd, temp_path_str = tempfile.mkstemp(suffix=suffix)\n resource_path = Path(temp_path_str)\n os.close(fd)\n if content:\n resource_path.write_bytes(content)\n log.reason(\"Created temporary file\", payload={\"path\": str(resource_path)})\n yield resource_path\n finally:\n if resource_path and resource_path.exists() and not dry_run:\n try:\n if resource_path.is_dir():\n shutil.rmtree(resource_path)\n log.reflect(\"Cleaned up temporary directory\", payload={\"path\": str(resource_path)})\n else:\n resource_path.unlink()\n log.reflect(\"Cleaned up temporary file\", payload={\"path\": str(resource_path)})\n except OSError as e:\n log.explore(\"Error during temp resource cleanup\", payload={\"path\": str(resource_path)}, error=str(e))\n# [/DEF:create_temp_file:Function]\n\n# [DEF:remove_empty_directories:Function]\n# @PURPOSE: Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.\n# @PRE: root_dir должен быть путем к существующей директории.\n# @POST: Все пустые поддиректории удалены, возвращено их количество.\n# @PARAM: root_dir (str) - Путь к корневой директории для очистки.\n# @RETURN: int - Количество удаленных директорий.\ndef remove_empty_directories(root_dir: str) -> int:\n with belief_scope(f\"Remove empty directories in {root_dir}\"):\n removed_count = 0\n if not os.path.isdir(root_dir):\n log.explore(\"Empty directory cleanup skipped - directory not found\", payload={\"root_dir\": root_dir}, error=f\"Directory not found: {root_dir}\")\n return 0\n for current_dir, _, _ in os.walk(root_dir, topdown=False):\n if not os.listdir(current_dir):\n try:\n os.rmdir(current_dir)\n removed_count += 1\n log.reason(\"Removed empty directory\", payload={\"directory\": current_dir})\n except OSError as e:\n log.explore(\"Failed to remove empty directory\", payload={\"directory\": current_dir}, error=str(e))\n log.reflect(\"Empty directory cleanup complete\", payload={\"removed_count\": removed_count})\n return removed_count\n# [/DEF:remove_empty_directories:Function]\n\n# [DEF:read_dashboard_from_disk:Function]\n# @PURPOSE: Читает бинарное содержимое файла с диска.\n# @PRE: file_path должен указывать на существующий файл.\n# @POST: Возвращает байты содержимого и имя файла.\n# @PARAM: file_path (str) - Путь к файлу.\n# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла).\n# @THROW: FileNotFoundError - Если файл не найден.\ndef read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:\n with belief_scope(f\"Read dashboard from {file_path}\"):\n path = Path(file_path)\n assert path.is_file(), f\"Файл дашборда не найден: {file_path}\"\n log.reason(\"Reading dashboard file from disk\", payload={\"file_path\": file_path})\n content = path.read_bytes()\n if not content:\n log.explore(\"Dashboard file is empty\", payload={\"file_path\": file_path}, error=\"Empty dashboard file\")\n return content, path.name\n# [/DEF:read_dashboard_from_disk:Function]\n\n# [DEF:calculate_crc32:Function]\n# @PURPOSE: Вычисляет контрольную сумму CRC32 для файла.\n# @PRE: file_path должен быть объектом Path к существующему файлу.\n# @POST: Возвращает 8-значную hex-строку CRC32.\n# @PARAM: file_path (Path) - Путь к файлу.\n# @RETURN: str - 8-значное шестнадцатеричное представление CRC32.\n# @THROW: IOError - При ошибках чтения файла.\ndef calculate_crc32(file_path: Path) -> str:\n with belief_scope(f\"Calculate CRC32 for {file_path}\"):\n with open(file_path, 'rb') as f:\n crc32_value = zlib.crc32(f.read())\n return f\"{crc32_value:08x}\"\n# [/DEF:calculate_crc32:Function]\n\n# [SECTION: DATA_CLASSES]\n# [DEF:RetentionPolicy:DataClass]\n# @PURPOSE: Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).\n@dataclass\nclass RetentionPolicy:\n daily: int = 7\n weekly: int = 4\n monthly: int = 12\n# [/DEF:RetentionPolicy:DataClass]\n# [/SECTION]\n\n# [DEF:archive_exports:Function]\n# @PURPOSE: Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.\n# @PRE: output_dir должен быть путем к существующей директории.\n# @POST: Старые или дублирующиеся архивы удалены согласно политике.\n# @RELATION: CALLS -> apply_retention_policy\n# @RELATION: CALLS -> calculate_crc32\n# @PARAM: output_dir (str) - Директория с архивами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32.\ndef archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:\n with belief_scope(f\"Archive exports in {output_dir}\"):\n output_path = Path(output_dir)\n if not output_path.is_dir():\n log.explore(\"Archive directory not found\", payload={\"output_dir\": output_dir}, error=f\"Archive directory not found: {output_dir}\")\n return\n\n log.reason(\"Managing archive exports\", payload={\"output_dir\": output_dir})\n \n # 1. Collect all zip files\n zip_files = list(output_path.glob(\"*.zip\"))\n if not zip_files:\n log.reason(\"No zip files found in archive directory\", payload={\"output_dir\": output_dir})\n return\n\n # 2. Deduplication\n if deduplicate:\n log.reason(\"Starting archive deduplication\")\n checksums = {}\n files_to_remove = []\n \n # Sort by modification time (newest first) to keep the latest version\n zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)\n \n for file_path in zip_files:\n try:\n crc = calculate_crc32(file_path)\n if crc in checksums:\n files_to_remove.append(file_path)\n log.reason(\"Duplicate archive found (same checksum)\", payload={\"file\": file_path.name, \"original\": checksums[crc].name})\n else:\n checksums[crc] = file_path\n except Exception as e:\n log.explore(\"Failed to calculate CRC32 for archive\", payload={\"file\": str(file_path)}, error=str(e))\n \n for f in files_to_remove:\n try:\n f.unlink()\n zip_files.remove(f)\n log.reason(\"Removed duplicate archive\", payload={\"file\": f.name})\n except OSError as e:\n log.explore(\"Failed to remove duplicate archive\", payload={\"file\": str(f)}, error=str(e))\n\n # 3. Retention Policy\n files_with_dates = []\n for file_path in zip_files:\n # Try to extract date from filename\n # Pattern: ..._YYYYMMDD_HHMMSS.zip or ..._YYYYMMDD.zip\n match = re.search(r'_(\\d{8})_', file_path.name)\n file_date = None\n if match:\n try:\n date_str = match.group(1)\n file_date = datetime.strptime(date_str, \"%Y%m%d\").date()\n except ValueError:\n pass\n \n if not file_date:\n # Fallback to modification time\n file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date()\n \n files_with_dates.append((file_path, file_date))\n\n files_to_keep = apply_retention_policy(files_with_dates, policy)\n \n for file_path, _ in files_with_dates:\n if file_path not in files_to_keep:\n try:\n file_path.unlink()\n log.reason(\"Removed archive by retention policy\", payload={\"file\": file_path.name})\n except OSError as e:\n log.explore(\"Failed to remove archive by retention policy\", payload={\"file\": str(file_path)}, error=str(e))\n# [/DEF:archive_exports:Function]\n\n# [DEF:apply_retention_policy:Function]\n# @PURPOSE: (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.\n# @PRE: files_with_dates is a list of (Path, date) tuples.\n# @POST: Returns a set of files to keep.\n# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @RETURN: set - Множество путей к файлам, которые должны быть сохранены.\ndef apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:\n with belief_scope(\"Apply retention policy\"):\n # Сортируем по дате (от новой к старой)\n sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True) \n # Словарь для хранения файлов по категориям \n daily_files = [] \n weekly_files = [] \n monthly_files = [] \n today = date.today() \n for file_path, file_date in sorted_files: \n # Ежедневные \n if (today - file_date).days < policy.daily: \n daily_files.append(file_path) \n # Еженедельные \n elif (today - file_date).days < policy.weekly * 7: \n weekly_files.append(file_path) \n # Ежемесячные \n elif (today - file_date).days < policy.monthly * 30: \n monthly_files.append(file_path) \n # Возвращаем множество файлов, которые нужно сохранить \n files_to_keep = set() \n files_to_keep.update(daily_files) \n files_to_keep.update(weekly_files[:policy.weekly]) \n files_to_keep.update(monthly_files[:policy.monthly]) \n log.reflect(\"Retention policy applied\", payload={\"files_to_keep\": len(files_to_keep)})\n return files_to_keep\n# [/DEF:apply_retention_policy:Function]\n\n# [DEF:save_and_unpack_dashboard:Function]\n# @PURPOSE: Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.\n# @PRE: zip_content должен быть байтами валидного ZIP-архива.\n# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.\n# @PARAM: zip_content (bytes) - Содержимое ZIP-архива.\n# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения.\n# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив.\n# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения.\n# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой.\n# @THROW: InvalidZipFormatError - При ошибке формата ZIP.\ndef save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:\n with belief_scope(\"Save and unpack dashboard\"):\n log.reason(\"Processing dashboard ZIP save\", payload={\"unpack\": unpack})\n try:\n output_path = Path(output_dir)\n output_path.mkdir(parents=True, exist_ok=True)\n zip_name = sanitize_filename(original_filename) if original_filename else f\"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip\"\n zip_path = output_path / zip_name\n zip_path.write_bytes(zip_content)\n log.reason(\"Dashboard ZIP saved to disk\", payload={\"zip_path\": str(zip_path)})\n if unpack:\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(output_path)\n log.reason(\"Dashboard unpacked to directory\", payload={\"output_path\": str(output_path)})\n return zip_path, output_path\n return zip_path, None\n except zipfile.BadZipFile as e:\n log.explore(\"Invalid ZIP archive format\", payload={\"output_dir\": str(output_dir)}, error=str(e))\n raise InvalidZipFormatError(f\"Invalid ZIP file: {e}\") from e\n# [/DEF:save_and_unpack_dashboard:Function]\n\n# [DEF:update_yamls:Function]\n# @PURPOSE: Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.\n# @PRE: path должен быть существующей директорией.\n# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.\n# @RELATION: CALLS -> _update_yaml_file\n# @THROW: FileNotFoundError - Если `path` не существует.\n# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены.\n# @PARAM: path (str) - Путь к директории с YAML файлами.\n# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска.\n# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены.\ndef update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = \"dashboards\", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:\n with belief_scope(\"Update YAML configurations\"):\n log.reason(\"Starting YAML configuration update\", payload={\"path\": path})\n dir_path = Path(path)\n assert dir_path.is_dir(), f\"Путь {path} не существует или не является директорией\"\n \n configs: List[Dict[str, Any]] = db_configs or []\n \n for file_path in dir_path.rglob(\"*.yaml\"):\n _update_yaml_file(file_path, configs, regexp_pattern, replace_string)\n# [/DEF:update_yamls:Function]\n\n# [DEF:_update_yaml_file:Function]\n# @PURPOSE: (Helper) Обновляет один YAML файл.\n# @PRE: file_path должен быть объектом Path к существующему YAML файлу.\n# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.\n# @PARAM: file_path (Path) - Путь к файлу.\n# @PARAM: db_configs (List[Dict]) - Конфигурации.\n# @PARAM: regexp_pattern (Optional[str]) - Паттерн.\n# @PARAM: replace_string (Optional[str]) - Замена.\ndef _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:\n with belief_scope(f\"Update YAML file: {file_path}\"):\n # Читаем содержимое файла\n try:\n with open(file_path, 'r', encoding='utf-8') as f:\n content = f.read()\n except Exception as e:\n log.explore(\"Failed to read YAML file\", payload={\"file_path\": str(file_path)}, error=str(e))\n return\n # Если задан pattern и replace_string, применяем замену по регулярному выражению\n if regexp_pattern and replace_string:\n try:\n new_content = re.sub(regexp_pattern, replace_string, content)\n if new_content != content:\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(new_content)\n log.reason(\"Updated YAML file using regex pattern\", payload={\"file_path\": str(file_path)})\n except Exception as e:\n log.explore(\"Error applying regex to YAML file\", payload={\"file_path\": str(file_path)}, error=str(e))\n # Если заданы конфигурации, заменяем значения (поддержка old/new)\n if db_configs:\n try:\n # Прямой текстовый заменитель для старых/новых значений, чтобы сохранить структуру файла\n modified_content = content\n for cfg in db_configs:\n # Ожидаем структуру: {'old': {...}, 'new': {...}}\n old_cfg = cfg.get('old', {})\n new_cfg = cfg.get('new', {})\n for key, old_val in old_cfg.items():\n if key in new_cfg:\n new_val = new_cfg[key]\n # Заменяем только точные совпадения старого значения в тексте YAML, используя ключ для контекста\n if isinstance(old_val, str):\n # Ищем паттерн: key: \"value\" или key: value\n key_pattern = re.escape(key)\n val_pattern = re.escape(old_val)\n # Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)\n pattern = rf'({key_pattern}\\s*:\\s*)([\"\\']?)({val_pattern})([\"\\']?)'\n \n # [DEF:replacer:Function]\n # @PURPOSE: Функция замены, сохраняющая кавычки если они были.\n # @PRE: match должен быть объектом совпадения регулярного выражения.\n # @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.\n def replacer(match):\n prefix = match.group(1)\n quote_open = match.group(2)\n quote_close = match.group(4)\n return f\"{prefix}{quote_open}{new_val}{quote_close}\"\n # [/DEF:replacer:Function]\n\n modified_content = re.sub(pattern, replacer, modified_content)\n log.reason(\"Replaced YAML config value\", payload={\"key\": key, \"file_path\": str(file_path)})\n # Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(modified_content)\n except Exception as e:\n log.explore(\"Error performing raw replacement in YAML\", payload={\"file_path\": str(file_path)}, error=str(e))\n# [/DEF:_update_yaml_file:Function]\n\n# [DEF:create_dashboard_export:Function]\n# @PURPOSE: Создает ZIP-архив из указанных исходных путей.\n# @PRE: source_paths должен содержать существующие пути.\n# @POST: ZIP-архив создан по пути zip_path.\n# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива.\n# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации.\n# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения.\n# @RETURN: bool - `True` при успехе, `False` при ошибке.\ndef create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:\n with belief_scope(f\"Create dashboard export: {zip_path}\"):\n log.reason(\"Packing dashboard export\", payload={\"source_paths\": [str(p) for p in source_paths], \"zip_path\": str(zip_path)})\n try:\n exclude_ext = [ext.lower() for ext in exclude_extensions or []]\n with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n for src_path_str in source_paths:\n src_path = Path(src_path_str)\n assert src_path.exists(), f\"Путь не найден: {src_path}\"\n for item in src_path.rglob('*'):\n if item.is_file() and item.suffix.lower() not in exclude_ext:\n arcname = item.relative_to(src_path.parent)\n zipf.write(item, arcname)\n log.reflect(\"Dashboard archive created\", payload={\"zip_path\": str(zip_path)})\n return True\n except (IOError, zipfile.BadZipFile, AssertionError) as e:\n log.explore(\"Dashboard archive creation failed\", payload={\"zip_path\": str(zip_path)}, error=str(e))\n return False\n# [/DEF:create_dashboard_export:Function]\n\n# [DEF:sanitize_filename:Function]\n# @PURPOSE: Очищает строку от символов, недопустимых в именах файлов.\n# @PRE: filename должен быть строкой.\n# @POST: Возвращает строку без спецсимволов.\n# @PARAM: filename (str) - Исходное имя файла.\n# @RETURN: str - Очищенная строка.\ndef sanitize_filename(filename: str) -> str:\n with belief_scope(f\"Sanitize filename: {filename}\"):\n return re.sub(r'[\\\\/*?:\"<>|]', \"_\", filename).strip()\n# [/DEF:sanitize_filename:Function]\n\n# [DEF:get_filename_from_headers:Function]\n# @PURPOSE: Извлекает имя файла из HTTP заголовка 'Content-Disposition'.\n# @PRE: headers должен быть словарем заголовков.\n# @POST: Возвращает имя файла или None, если заголовок отсутствует.\n# @PARAM: headers (dict) - Словарь HTTP заголовков.\n# @RETURN: Optional[str] - Имя файла or `None`.\ndef get_filename_from_headers(headers: dict) -> Optional[str]:\n with belief_scope(\"Get filename from headers\"):\n content_disposition = headers.get(\"Content-Disposition\", \"\")\n if match := re.search(r'filename=\"?([^\"]+)\"?', content_disposition):\n return match.group(1).strip()\n return None\n# [/DEF:get_filename_from_headers:Function]\n\n# [DEF:consolidate_archive_folders:Function]\n# @PURPOSE: Консолидирует директории архивов на основе общего слага в имени.\n# @PRE: root_directory должен быть объектом Path к существующей директории.\n# @POST: Директории с одинаковым префиксом объединены в одну.\n# @THROW: TypeError, ValueError - Если `root_directory` невалиден.\n# @PARAM: root_directory (Path) - Корневая директория для консолидации.\ndef consolidate_archive_folders(root_directory: Path) -> None:\n with belief_scope(f\"Consolidate archives in {root_directory}\"):\n assert isinstance(root_directory, Path), \"root_directory must be a Path object.\" \n assert root_directory.is_dir(), \"root_directory must be an existing directory.\" \n \n log.reason(\"Consolidating archive folders\", payload={\"root_directory\": str(root_directory)}) \n # Собираем все директории с архивами \n archive_dirs = [] \n for item in root_directory.iterdir(): \n if item.is_dir(): \n # Проверяем, есть ли в директории ZIP-архивы \n if any(item.glob(\"*.zip\")): \n archive_dirs.append(item) \n # Группируем по слагу (части имени до первого '_') \n slug_groups = {} \n for dir_path in archive_dirs: \n dir_name = dir_path.name \n slug = dir_name.split('_')[0] if '_' in dir_name else dir_name \n if slug not in slug_groups: \n slug_groups[slug] = [] \n slug_groups[slug].append(dir_path) \n # Для каждой группы консолидируем \n for slug, dirs in slug_groups.items(): \n if len(dirs) <= 1: \n continue \n # Создаем целевую директорию \n target_dir = root_directory / slug \n target_dir.mkdir(exist_ok=True) \n log.reason(\"Consolidating archive directories\", payload={\"slug\": slug, \"dir_count\": len(dirs), \"target\": str(target_dir)}) \n # Перемещаем содержимое \n for source_dir in dirs: \n if source_dir == target_dir: \n continue \n for item in source_dir.iterdir(): \n dest_item = target_dir / item.name \n try: \n if item.is_dir(): \n shutil.move(str(item), str(dest_item)) \n else: \n shutil.move(str(item), str(dest_item)) \n except Exception as e: \n log.explore(\"Failed to move item during consolidation\", payload={\"source\": str(item), \"dest\": str(dest_item)}, error=str(e)) \n # Удаляем исходную директорию \n try: \n source_dir.rmdir() \n log.reason(\"Removed source directory after consolidation\", payload={\"directory\": str(source_dir)}) \n except Exception as e: \n log.explore(\"Failed to remove source directory after consolidation\", payload={\"directory\": str(source_dir)}, error=str(e)) \n# [/DEF:consolidate_archive_folders:Function]\n\n# [/DEF:FileIO:Module]\n" }, { "contract_id": "InvalidZipFormatError", "contract_type": "Class", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 25, - "end_line": 29, + "start_line": 28, + "end_line": 32, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -48958,8 +49714,8 @@ "contract_id": "create_temp_file", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 31, - "end_line": 70, + "start_line": 34, + "end_line": 73, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49010,14 +49766,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:create_temp_file:Function]\n# @PURPOSE: Контекстный менеджер для создания временного файла или директории с гарантированным удалением.\n# @PRE: suffix должен быть строкой, определяющей тип ресурса.\n# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.\n# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл.\n# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория.\n# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb').\n# @YIELDS: Path - Путь к временному ресурсу.\n# @THROW: IOError - При ошибках создания ресурса.\n@contextmanager\ndef create_temp_file(content: Optional[bytes] = None, suffix: str = \".zip\", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:\n with belief_scope(\"Create temporary resource\"):\n resource_path = None\n is_dir = suffix.startswith('.dir')\n try:\n if is_dir:\n with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir:\n resource_path = Path(temp_dir)\n app_logger.debug(\"[create_temp_file][State] Created temporary directory: %s\", resource_path)\n yield resource_path\n else:\n fd, temp_path_str = tempfile.mkstemp(suffix=suffix)\n resource_path = Path(temp_path_str)\n os.close(fd)\n if content:\n resource_path.write_bytes(content)\n app_logger.debug(\"[create_temp_file][State] Created temporary file: %s\", resource_path)\n yield resource_path\n finally:\n if resource_path and resource_path.exists() and not dry_run:\n try:\n if resource_path.is_dir():\n shutil.rmtree(resource_path)\n app_logger.debug(\"[create_temp_file][Cleanup] Removed temporary directory: %s\", resource_path)\n else:\n resource_path.unlink()\n app_logger.debug(\"[create_temp_file][Cleanup] Removed temporary file: %s\", resource_path)\n except OSError as e:\n app_logger.error(\"[create_temp_file][Failure] Error during cleanup of %s: %s\", resource_path, e)\n# [/DEF:create_temp_file:Function]\n" + "body": "# [DEF:create_temp_file:Function]\n# @PURPOSE: Контекстный менеджер для создания временного файла или директории с гарантированным удалением.\n# @PRE: suffix должен быть строкой, определяющей тип ресурса.\n# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.\n# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл.\n# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория.\n# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb').\n# @YIELDS: Path - Путь к временному ресурсу.\n# @THROW: IOError - При ошибках создания ресурса.\n@contextmanager\ndef create_temp_file(content: Optional[bytes] = None, suffix: str = \".zip\", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:\n with belief_scope(\"Create temporary resource\"):\n resource_path = None\n is_dir = suffix.startswith('.dir')\n try:\n if is_dir:\n with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir:\n resource_path = Path(temp_dir)\n log.reason(\"Created temporary directory\", payload={\"path\": str(resource_path)})\n yield resource_path\n else:\n fd, temp_path_str = tempfile.mkstemp(suffix=suffix)\n resource_path = Path(temp_path_str)\n os.close(fd)\n if content:\n resource_path.write_bytes(content)\n log.reason(\"Created temporary file\", payload={\"path\": str(resource_path)})\n yield resource_path\n finally:\n if resource_path and resource_path.exists() and not dry_run:\n try:\n if resource_path.is_dir():\n shutil.rmtree(resource_path)\n log.reflect(\"Cleaned up temporary directory\", payload={\"path\": str(resource_path)})\n else:\n resource_path.unlink()\n log.reflect(\"Cleaned up temporary file\", payload={\"path\": str(resource_path)})\n except OSError as e:\n log.explore(\"Error during temp resource cleanup\", payload={\"path\": str(resource_path)}, error=str(e))\n# [/DEF:create_temp_file:Function]\n" }, { "contract_id": "remove_empty_directories", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 72, - "end_line": 95, + "start_line": 75, + "end_line": 97, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49061,14 +49817,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:remove_empty_directories:Function]\n# @PURPOSE: Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.\n# @PRE: root_dir должен быть путем к существующей директории.\n# @POST: Все пустые поддиректории удалены, возвращено их количество.\n# @PARAM: root_dir (str) - Путь к корневой директории для очистки.\n# @RETURN: int - Количество удаленных директорий.\ndef remove_empty_directories(root_dir: str) -> int:\n with belief_scope(f\"Remove empty directories in {root_dir}\"):\n app_logger.info(\"[remove_empty_directories][Enter] Starting cleanup of empty directories in %s\", root_dir)\n removed_count = 0\n if not os.path.isdir(root_dir):\n app_logger.error(\"[remove_empty_directories][Failure] Directory not found: %s\", root_dir)\n return 0\n for current_dir, _, _ in os.walk(root_dir, topdown=False):\n if not os.listdir(current_dir):\n try:\n os.rmdir(current_dir)\n removed_count += 1\n app_logger.info(\"[remove_empty_directories][State] Removed empty directory: %s\", current_dir)\n except OSError as e:\n app_logger.error(\"[remove_empty_directories][Failure] Failed to remove %s: %s\", current_dir, e)\n app_logger.info(\"[remove_empty_directories][Exit] Removed %d empty directories.\", removed_count)\n return removed_count\n# [/DEF:remove_empty_directories:Function]\n" + "body": "# [DEF:remove_empty_directories:Function]\n# @PURPOSE: Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.\n# @PRE: root_dir должен быть путем к существующей директории.\n# @POST: Все пустые поддиректории удалены, возвращено их количество.\n# @PARAM: root_dir (str) - Путь к корневой директории для очистки.\n# @RETURN: int - Количество удаленных директорий.\ndef remove_empty_directories(root_dir: str) -> int:\n with belief_scope(f\"Remove empty directories in {root_dir}\"):\n removed_count = 0\n if not os.path.isdir(root_dir):\n log.explore(\"Empty directory cleanup skipped - directory not found\", payload={\"root_dir\": root_dir}, error=f\"Directory not found: {root_dir}\")\n return 0\n for current_dir, _, _ in os.walk(root_dir, topdown=False):\n if not os.listdir(current_dir):\n try:\n os.rmdir(current_dir)\n removed_count += 1\n log.reason(\"Removed empty directory\", payload={\"directory\": current_dir})\n except OSError as e:\n log.explore(\"Failed to remove empty directory\", payload={\"directory\": current_dir}, error=str(e))\n log.reflect(\"Empty directory cleanup complete\", payload={\"removed_count\": removed_count})\n return removed_count\n# [/DEF:remove_empty_directories:Function]\n" }, { "contract_id": "read_dashboard_from_disk", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 97, - "end_line": 113, + "start_line": 99, + "end_line": 115, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49119,14 +49875,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:read_dashboard_from_disk:Function]\n# @PURPOSE: Читает бинарное содержимое файла с диска.\n# @PRE: file_path должен указывать на существующий файл.\n# @POST: Возвращает байты содержимого и имя файла.\n# @PARAM: file_path (str) - Путь к файлу.\n# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла).\n# @THROW: FileNotFoundError - Если файл не найден.\ndef read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:\n with belief_scope(f\"Read dashboard from {file_path}\"):\n path = Path(file_path)\n assert path.is_file(), f\"Файл дашборда не найден: {file_path}\"\n app_logger.info(\"[read_dashboard_from_disk][Enter] Reading file: %s\", file_path)\n content = path.read_bytes()\n if not content:\n app_logger.warning(\"[read_dashboard_from_disk][Warning] File is empty: %s\", file_path)\n return content, path.name\n# [/DEF:read_dashboard_from_disk:Function]\n" + "body": "# [DEF:read_dashboard_from_disk:Function]\n# @PURPOSE: Читает бинарное содержимое файла с диска.\n# @PRE: file_path должен указывать на существующий файл.\n# @POST: Возвращает байты содержимого и имя файла.\n# @PARAM: file_path (str) - Путь к файлу.\n# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла).\n# @THROW: FileNotFoundError - Если файл не найден.\ndef read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:\n with belief_scope(f\"Read dashboard from {file_path}\"):\n path = Path(file_path)\n assert path.is_file(), f\"Файл дашборда не найден: {file_path}\"\n log.reason(\"Reading dashboard file from disk\", payload={\"file_path\": file_path})\n content = path.read_bytes()\n if not content:\n log.explore(\"Dashboard file is empty\", payload={\"file_path\": file_path}, error=\"Empty dashboard file\")\n return content, path.name\n# [/DEF:read_dashboard_from_disk:Function]\n" }, { "contract_id": "calculate_crc32", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 115, - "end_line": 127, + "start_line": 117, + "end_line": 129, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49183,8 +49939,8 @@ "contract_id": "RetentionPolicy", "contract_type": "DataClass", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 130, - "end_line": 137, + "start_line": 132, + "end_line": 139, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49225,8 +49981,8 @@ "contract_id": "archive_exports", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 140, - "end_line": 221, + "start_line": 142, + "end_line": 223, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49285,14 +50041,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:archive_exports:Function]\n# @PURPOSE: Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.\n# @PRE: output_dir должен быть путем к существующей директории.\n# @POST: Старые или дублирующиеся архивы удалены согласно политике.\n# @RELATION: CALLS -> apply_retention_policy\n# @RELATION: CALLS -> calculate_crc32\n# @PARAM: output_dir (str) - Директория с архивами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32.\ndef archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:\n with belief_scope(f\"Archive exports in {output_dir}\"):\n output_path = Path(output_dir)\n if not output_path.is_dir():\n app_logger.warning(\"[archive_exports][Skip] Archive directory not found: %s\", output_dir)\n return\n\n app_logger.info(\"[archive_exports][Enter] Managing archive in %s\", output_dir)\n \n # 1. Collect all zip files\n zip_files = list(output_path.glob(\"*.zip\"))\n if not zip_files:\n app_logger.info(\"[archive_exports][State] No zip files found in %s\", output_dir)\n return\n\n # 2. Deduplication\n if deduplicate:\n app_logger.info(\"[archive_exports][State] Starting deduplication...\")\n checksums = {}\n files_to_remove = []\n \n # Sort by modification time (newest first) to keep the latest version\n zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)\n \n for file_path in zip_files:\n try:\n crc = calculate_crc32(file_path)\n if crc in checksums:\n files_to_remove.append(file_path)\n app_logger.debug(\"[archive_exports][State] Duplicate found: %s (same as %s)\", file_path.name, checksums[crc].name)\n else:\n checksums[crc] = file_path\n except Exception as e:\n app_logger.error(\"[archive_exports][Failure] Failed to calculate CRC32 for %s: %s\", file_path, e)\n \n for f in files_to_remove:\n try:\n f.unlink()\n zip_files.remove(f)\n app_logger.info(\"[archive_exports][State] Removed duplicate: %s\", f.name)\n except OSError as e:\n app_logger.error(\"[archive_exports][Failure] Failed to remove duplicate %s: %s\", f, e)\n\n # 3. Retention Policy\n files_with_dates = []\n for file_path in zip_files:\n # Try to extract date from filename\n # Pattern: ..._YYYYMMDD_HHMMSS.zip or ..._YYYYMMDD.zip\n match = re.search(r'_(\\d{8})_', file_path.name)\n file_date = None\n if match:\n try:\n date_str = match.group(1)\n file_date = datetime.strptime(date_str, \"%Y%m%d\").date()\n except ValueError:\n pass\n \n if not file_date:\n # Fallback to modification time\n file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date()\n \n files_with_dates.append((file_path, file_date))\n\n files_to_keep = apply_retention_policy(files_with_dates, policy)\n \n for file_path, _ in files_with_dates:\n if file_path not in files_to_keep:\n try:\n file_path.unlink()\n app_logger.info(\"[archive_exports][State] Removed by retention policy: %s\", file_path.name)\n except OSError as e:\n app_logger.error(\"[archive_exports][Failure] Failed to remove %s: %s\", file_path, e)\n# [/DEF:archive_exports:Function]\n" + "body": "# [DEF:archive_exports:Function]\n# @PURPOSE: Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.\n# @PRE: output_dir должен быть путем к существующей директории.\n# @POST: Старые или дублирующиеся архивы удалены согласно политике.\n# @RELATION: CALLS -> apply_retention_policy\n# @RELATION: CALLS -> calculate_crc32\n# @PARAM: output_dir (str) - Директория с архивами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32.\ndef archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:\n with belief_scope(f\"Archive exports in {output_dir}\"):\n output_path = Path(output_dir)\n if not output_path.is_dir():\n log.explore(\"Archive directory not found\", payload={\"output_dir\": output_dir}, error=f\"Archive directory not found: {output_dir}\")\n return\n\n log.reason(\"Managing archive exports\", payload={\"output_dir\": output_dir})\n \n # 1. Collect all zip files\n zip_files = list(output_path.glob(\"*.zip\"))\n if not zip_files:\n log.reason(\"No zip files found in archive directory\", payload={\"output_dir\": output_dir})\n return\n\n # 2. Deduplication\n if deduplicate:\n log.reason(\"Starting archive deduplication\")\n checksums = {}\n files_to_remove = []\n \n # Sort by modification time (newest first) to keep the latest version\n zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)\n \n for file_path in zip_files:\n try:\n crc = calculate_crc32(file_path)\n if crc in checksums:\n files_to_remove.append(file_path)\n log.reason(\"Duplicate archive found (same checksum)\", payload={\"file\": file_path.name, \"original\": checksums[crc].name})\n else:\n checksums[crc] = file_path\n except Exception as e:\n log.explore(\"Failed to calculate CRC32 for archive\", payload={\"file\": str(file_path)}, error=str(e))\n \n for f in files_to_remove:\n try:\n f.unlink()\n zip_files.remove(f)\n log.reason(\"Removed duplicate archive\", payload={\"file\": f.name})\n except OSError as e:\n log.explore(\"Failed to remove duplicate archive\", payload={\"file\": str(f)}, error=str(e))\n\n # 3. Retention Policy\n files_with_dates = []\n for file_path in zip_files:\n # Try to extract date from filename\n # Pattern: ..._YYYYMMDD_HHMMSS.zip or ..._YYYYMMDD.zip\n match = re.search(r'_(\\d{8})_', file_path.name)\n file_date = None\n if match:\n try:\n date_str = match.group(1)\n file_date = datetime.strptime(date_str, \"%Y%m%d\").date()\n except ValueError:\n pass\n \n if not file_date:\n # Fallback to modification time\n file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date()\n \n files_with_dates.append((file_path, file_date))\n\n files_to_keep = apply_retention_policy(files_with_dates, policy)\n \n for file_path, _ in files_with_dates:\n if file_path not in files_to_keep:\n try:\n file_path.unlink()\n log.reason(\"Removed archive by retention policy\", payload={\"file\": file_path.name})\n except OSError as e:\n log.explore(\"Failed to remove archive by retention policy\", payload={\"file\": str(file_path)}, error=str(e))\n# [/DEF:archive_exports:Function]\n" }, { "contract_id": "apply_retention_policy", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 223, - "end_line": 256, + "start_line": 225, + "end_line": 258, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49336,14 +50092,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:apply_retention_policy:Function]\n# @PURPOSE: (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.\n# @PRE: files_with_dates is a list of (Path, date) tuples.\n# @POST: Returns a set of files to keep.\n# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @RETURN: set - Множество путей к файлам, которые должны быть сохранены.\ndef apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:\n with belief_scope(\"Apply retention policy\"):\n # Сортируем по дате (от новой к старой)\n sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True) \n # Словарь для хранения файлов по категориям \n daily_files = [] \n weekly_files = [] \n monthly_files = [] \n today = date.today() \n for file_path, file_date in sorted_files: \n # Ежедневные \n if (today - file_date).days < policy.daily: \n daily_files.append(file_path) \n # Еженедельные \n elif (today - file_date).days < policy.weekly * 7: \n weekly_files.append(file_path) \n # Ежемесячные \n elif (today - file_date).days < policy.monthly * 30: \n monthly_files.append(file_path) \n # Возвращаем множество файлов, которые нужно сохранить \n files_to_keep = set() \n files_to_keep.update(daily_files) \n files_to_keep.update(weekly_files[:policy.weekly]) \n files_to_keep.update(monthly_files[:policy.monthly]) \n app_logger.debug(\"[apply_retention_policy][State] Keeping %d files according to retention policy\", len(files_to_keep))\n return files_to_keep\n# [/DEF:apply_retention_policy:Function]\n" + "body": "# [DEF:apply_retention_policy:Function]\n# @PURPOSE: (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.\n# @PRE: files_with_dates is a list of (Path, date) tuples.\n# @POST: Returns a set of files to keep.\n# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами.\n# @PARAM: policy (RetentionPolicy) - Политика хранения.\n# @RETURN: set - Множество путей к файлам, которые должны быть сохранены.\ndef apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:\n with belief_scope(\"Apply retention policy\"):\n # Сортируем по дате (от новой к старой)\n sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True) \n # Словарь для хранения файлов по категориям \n daily_files = [] \n weekly_files = [] \n monthly_files = [] \n today = date.today() \n for file_path, file_date in sorted_files: \n # Ежедневные \n if (today - file_date).days < policy.daily: \n daily_files.append(file_path) \n # Еженедельные \n elif (today - file_date).days < policy.weekly * 7: \n weekly_files.append(file_path) \n # Ежемесячные \n elif (today - file_date).days < policy.monthly * 30: \n monthly_files.append(file_path) \n # Возвращаем множество файлов, которые нужно сохранить \n files_to_keep = set() \n files_to_keep.update(daily_files) \n files_to_keep.update(weekly_files[:policy.weekly]) \n files_to_keep.update(monthly_files[:policy.monthly]) \n log.reflect(\"Retention policy applied\", payload={\"files_to_keep\": len(files_to_keep)})\n return files_to_keep\n# [/DEF:apply_retention_policy:Function]\n" }, { "contract_id": "save_and_unpack_dashboard", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 258, - "end_line": 287, + "start_line": 260, + "end_line": 289, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49394,14 +50150,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:save_and_unpack_dashboard:Function]\n# @PURPOSE: Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.\n# @PRE: zip_content должен быть байтами валидного ZIP-архива.\n# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.\n# @PARAM: zip_content (bytes) - Содержимое ZIP-архива.\n# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения.\n# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив.\n# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения.\n# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой.\n# @THROW: InvalidZipFormatError - При ошибке формата ZIP.\ndef save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:\n with belief_scope(\"Save and unpack dashboard\"):\n app_logger.info(\"[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s\", unpack)\n try:\n output_path = Path(output_dir)\n output_path.mkdir(parents=True, exist_ok=True)\n zip_name = sanitize_filename(original_filename) if original_filename else f\"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip\"\n zip_path = output_path / zip_name\n zip_path.write_bytes(zip_content)\n app_logger.info(\"[save_and_unpack_dashboard][State] Dashboard saved to: %s\", zip_path)\n if unpack:\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(output_path)\n app_logger.info(\"[save_and_unpack_dashboard][State] Dashboard unpacked to: %s\", output_path)\n return zip_path, output_path\n return zip_path, None\n except zipfile.BadZipFile as e:\n app_logger.error(\"[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s\", e)\n raise InvalidZipFormatError(f\"Invalid ZIP file: {e}\") from e\n# [/DEF:save_and_unpack_dashboard:Function]\n" + "body": "# [DEF:save_and_unpack_dashboard:Function]\n# @PURPOSE: Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.\n# @PRE: zip_content должен быть байтами валидного ZIP-архива.\n# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.\n# @PARAM: zip_content (bytes) - Содержимое ZIP-архива.\n# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения.\n# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив.\n# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения.\n# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой.\n# @THROW: InvalidZipFormatError - При ошибке формата ZIP.\ndef save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:\n with belief_scope(\"Save and unpack dashboard\"):\n log.reason(\"Processing dashboard ZIP save\", payload={\"unpack\": unpack})\n try:\n output_path = Path(output_dir)\n output_path.mkdir(parents=True, exist_ok=True)\n zip_name = sanitize_filename(original_filename) if original_filename else f\"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip\"\n zip_path = output_path / zip_name\n zip_path.write_bytes(zip_content)\n log.reason(\"Dashboard ZIP saved to disk\", payload={\"zip_path\": str(zip_path)})\n if unpack:\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(output_path)\n log.reason(\"Dashboard unpacked to directory\", payload={\"output_path\": str(output_path)})\n return zip_path, output_path\n return zip_path, None\n except zipfile.BadZipFile as e:\n log.explore(\"Invalid ZIP archive format\", payload={\"output_dir\": str(output_dir)}, error=str(e))\n raise InvalidZipFormatError(f\"Invalid ZIP file: {e}\") from e\n# [/DEF:save_and_unpack_dashboard:Function]\n" }, { "contract_id": "update_yamls", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 289, - "end_line": 309, + "start_line": 291, + "end_line": 311, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49461,14 +50217,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:update_yamls:Function]\n# @PURPOSE: Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.\n# @PRE: path должен быть существующей директорией.\n# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.\n# @RELATION: CALLS -> _update_yaml_file\n# @THROW: FileNotFoundError - Если `path` не существует.\n# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены.\n# @PARAM: path (str) - Путь к директории с YAML файлами.\n# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска.\n# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены.\ndef update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = \"dashboards\", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:\n with belief_scope(\"Update YAML configurations\"):\n app_logger.info(\"[update_yamls][Enter] Starting YAML configuration update.\")\n dir_path = Path(path)\n assert dir_path.is_dir(), f\"Путь {path} не существует или не является директорией\"\n \n configs: List[Dict[str, Any]] = db_configs or []\n \n for file_path in dir_path.rglob(\"*.yaml\"):\n _update_yaml_file(file_path, configs, regexp_pattern, replace_string)\n# [/DEF:update_yamls:Function]\n" + "body": "# [DEF:update_yamls:Function]\n# @PURPOSE: Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.\n# @PRE: path должен быть существующей директорией.\n# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.\n# @RELATION: CALLS -> _update_yaml_file\n# @THROW: FileNotFoundError - Если `path` не существует.\n# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены.\n# @PARAM: path (str) - Путь к директории с YAML файлами.\n# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска.\n# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены.\ndef update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = \"dashboards\", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:\n with belief_scope(\"Update YAML configurations\"):\n log.reason(\"Starting YAML configuration update\", payload={\"path\": path})\n dir_path = Path(path)\n assert dir_path.is_dir(), f\"Путь {path} не существует или не является директорией\"\n \n configs: List[Dict[str, Any]] = db_configs or []\n \n for file_path in dir_path.rglob(\"*.yaml\"):\n _update_yaml_file(file_path, configs, regexp_pattern, replace_string)\n# [/DEF:update_yamls:Function]\n" }, { "contract_id": "_update_yaml_file", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 311, - "end_line": 376, + "start_line": 313, + "end_line": 378, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49505,14 +50261,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:_update_yaml_file:Function]\n# @PURPOSE: (Helper) Обновляет один YAML файл.\n# @PRE: file_path должен быть объектом Path к существующему YAML файлу.\n# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.\n# @PARAM: file_path (Path) - Путь к файлу.\n# @PARAM: db_configs (List[Dict]) - Конфигурации.\n# @PARAM: regexp_pattern (Optional[str]) - Паттерн.\n# @PARAM: replace_string (Optional[str]) - Замена.\ndef _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:\n with belief_scope(f\"Update YAML file: {file_path}\"):\n # Читаем содержимое файла\n try:\n with open(file_path, 'r', encoding='utf-8') as f:\n content = f.read()\n except Exception as e:\n app_logger.error(\"[_update_yaml_file][Failure] Failed to read %s: %s\", file_path, e)\n return\n # Если задан pattern и replace_string, применяем замену по регулярному выражению\n if regexp_pattern and replace_string:\n try:\n new_content = re.sub(regexp_pattern, replace_string, content)\n if new_content != content:\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(new_content)\n app_logger.info(\"[_update_yaml_file][State] Updated %s using regex pattern\", file_path)\n except Exception as e:\n app_logger.error(\"[_update_yaml_file][Failure] Error applying regex to %s: %s\", file_path, e)\n # Если заданы конфигурации, заменяем значения (поддержка old/new)\n if db_configs:\n try:\n # Прямой текстовый заменитель для старых/новых значений, чтобы сохранить структуру файла\n modified_content = content\n for cfg in db_configs:\n # Ожидаем структуру: {'old': {...}, 'new': {...}}\n old_cfg = cfg.get('old', {})\n new_cfg = cfg.get('new', {})\n for key, old_val in old_cfg.items():\n if key in new_cfg:\n new_val = new_cfg[key]\n # Заменяем только точные совпадения старого значения в тексте YAML, используя ключ для контекста\n if isinstance(old_val, str):\n # Ищем паттерн: key: \"value\" или key: value\n key_pattern = re.escape(key)\n val_pattern = re.escape(old_val)\n # Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)\n pattern = rf'({key_pattern}\\s*:\\s*)([\"\\']?)({val_pattern})([\"\\']?)'\n \n # [DEF:replacer:Function]\n # @PURPOSE: Функция замены, сохраняющая кавычки если они были.\n # @PRE: match должен быть объектом совпадения регулярного выражения.\n # @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.\n def replacer(match):\n prefix = match.group(1)\n quote_open = match.group(2)\n quote_close = match.group(4)\n return f\"{prefix}{quote_open}{new_val}{quote_close}\"\n # [/DEF:replacer:Function]\n\n modified_content = re.sub(pattern, replacer, modified_content)\n app_logger.info(\"[_update_yaml_file][State] Replaced '%s' with '%s' for key %s in %s\", old_val, new_val, key, file_path)\n # Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(modified_content)\n except Exception as e:\n app_logger.error(\"[_update_yaml_file][Failure] Error performing raw replacement in %s: %s\", file_path, e)\n# [/DEF:_update_yaml_file:Function]\n" + "body": "# [DEF:_update_yaml_file:Function]\n# @PURPOSE: (Helper) Обновляет один YAML файл.\n# @PRE: file_path должен быть объектом Path к существующему YAML файлу.\n# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.\n# @PARAM: file_path (Path) - Путь к файлу.\n# @PARAM: db_configs (List[Dict]) - Конфигурации.\n# @PARAM: regexp_pattern (Optional[str]) - Паттерн.\n# @PARAM: replace_string (Optional[str]) - Замена.\ndef _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:\n with belief_scope(f\"Update YAML file: {file_path}\"):\n # Читаем содержимое файла\n try:\n with open(file_path, 'r', encoding='utf-8') as f:\n content = f.read()\n except Exception as e:\n log.explore(\"Failed to read YAML file\", payload={\"file_path\": str(file_path)}, error=str(e))\n return\n # Если задан pattern и replace_string, применяем замену по регулярному выражению\n if regexp_pattern and replace_string:\n try:\n new_content = re.sub(regexp_pattern, replace_string, content)\n if new_content != content:\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(new_content)\n log.reason(\"Updated YAML file using regex pattern\", payload={\"file_path\": str(file_path)})\n except Exception as e:\n log.explore(\"Error applying regex to YAML file\", payload={\"file_path\": str(file_path)}, error=str(e))\n # Если заданы конфигурации, заменяем значения (поддержка old/new)\n if db_configs:\n try:\n # Прямой текстовый заменитель для старых/новых значений, чтобы сохранить структуру файла\n modified_content = content\n for cfg in db_configs:\n # Ожидаем структуру: {'old': {...}, 'new': {...}}\n old_cfg = cfg.get('old', {})\n new_cfg = cfg.get('new', {})\n for key, old_val in old_cfg.items():\n if key in new_cfg:\n new_val = new_cfg[key]\n # Заменяем только точные совпадения старого значения в тексте YAML, используя ключ для контекста\n if isinstance(old_val, str):\n # Ищем паттерн: key: \"value\" или key: value\n key_pattern = re.escape(key)\n val_pattern = re.escape(old_val)\n # Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)\n pattern = rf'({key_pattern}\\s*:\\s*)([\"\\']?)({val_pattern})([\"\\']?)'\n \n # [DEF:replacer:Function]\n # @PURPOSE: Функция замены, сохраняющая кавычки если они были.\n # @PRE: match должен быть объектом совпадения регулярного выражения.\n # @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.\n def replacer(match):\n prefix = match.group(1)\n quote_open = match.group(2)\n quote_close = match.group(4)\n return f\"{prefix}{quote_open}{new_val}{quote_close}\"\n # [/DEF:replacer:Function]\n\n modified_content = re.sub(pattern, replacer, modified_content)\n log.reason(\"Replaced YAML config value\", payload={\"key\": key, \"file_path\": str(file_path)})\n # Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование\n with open(file_path, 'w', encoding='utf-8') as f:\n f.write(modified_content)\n except Exception as e:\n log.explore(\"Error performing raw replacement in YAML\", payload={\"file_path\": str(file_path)}, error=str(e))\n# [/DEF:_update_yaml_file:Function]\n" }, { "contract_id": "replacer", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 358, - "end_line": 367, + "start_line": 360, + "end_line": 369, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49548,8 +50304,8 @@ "contract_id": "create_dashboard_export", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 378, - "end_line": 404, + "start_line": 380, + "end_line": 406, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49593,14 +50349,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:create_dashboard_export:Function]\n# @PURPOSE: Создает ZIP-архив из указанных исходных путей.\n# @PRE: source_paths должен содержать существующие пути.\n# @POST: ZIP-архив создан по пути zip_path.\n# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива.\n# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации.\n# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения.\n# @RETURN: bool - `True` при успехе, `False` при ошибке.\ndef create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:\n with belief_scope(f\"Create dashboard export: {zip_path}\"):\n app_logger.info(\"[create_dashboard_export][Enter] Packing dashboard: %s -> %s\", source_paths, zip_path)\n try:\n exclude_ext = [ext.lower() for ext in exclude_extensions or []]\n with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n for src_path_str in source_paths:\n src_path = Path(src_path_str)\n assert src_path.exists(), f\"Путь не найден: {src_path}\"\n for item in src_path.rglob('*'):\n if item.is_file() and item.suffix.lower() not in exclude_ext:\n arcname = item.relative_to(src_path.parent)\n zipf.write(item, arcname)\n app_logger.info(\"[create_dashboard_export][Exit] Archive created: %s\", zip_path)\n return True\n except (IOError, zipfile.BadZipFile, AssertionError) as e:\n app_logger.error(\"[create_dashboard_export][Failure] Error: %s\", e, exc_info=True)\n return False\n# [/DEF:create_dashboard_export:Function]\n" + "body": "# [DEF:create_dashboard_export:Function]\n# @PURPOSE: Создает ZIP-архив из указанных исходных путей.\n# @PRE: source_paths должен содержать существующие пути.\n# @POST: ZIP-архив создан по пути zip_path.\n# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива.\n# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации.\n# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения.\n# @RETURN: bool - `True` при успехе, `False` при ошибке.\ndef create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:\n with belief_scope(f\"Create dashboard export: {zip_path}\"):\n log.reason(\"Packing dashboard export\", payload={\"source_paths\": [str(p) for p in source_paths], \"zip_path\": str(zip_path)})\n try:\n exclude_ext = [ext.lower() for ext in exclude_extensions or []]\n with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n for src_path_str in source_paths:\n src_path = Path(src_path_str)\n assert src_path.exists(), f\"Путь не найден: {src_path}\"\n for item in src_path.rglob('*'):\n if item.is_file() and item.suffix.lower() not in exclude_ext:\n arcname = item.relative_to(src_path.parent)\n zipf.write(item, arcname)\n log.reflect(\"Dashboard archive created\", payload={\"zip_path\": str(zip_path)})\n return True\n except (IOError, zipfile.BadZipFile, AssertionError) as e:\n log.explore(\"Dashboard archive creation failed\", payload={\"zip_path\": str(zip_path)}, error=str(e))\n return False\n# [/DEF:create_dashboard_export:Function]\n" }, { "contract_id": "sanitize_filename", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 406, - "end_line": 415, + "start_line": 408, + "end_line": 417, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49650,8 +50406,8 @@ "contract_id": "get_filename_from_headers", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 417, - "end_line": 429, + "start_line": 419, + "end_line": 431, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49701,8 +50457,8 @@ "contract_id": "consolidate_archive_folders", "contract_type": "Function", "file_path": "backend/src/core/utils/fileio.py", - "start_line": 431, - "end_line": 485, + "start_line": 433, + "end_line": 487, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49746,7 +50502,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:consolidate_archive_folders:Function]\n# @PURPOSE: Консолидирует директории архивов на основе общего слага в имени.\n# @PRE: root_directory должен быть объектом Path к существующей директории.\n# @POST: Директории с одинаковым префиксом объединены в одну.\n# @THROW: TypeError, ValueError - Если `root_directory` невалиден.\n# @PARAM: root_directory (Path) - Корневая директория для консолидации.\ndef consolidate_archive_folders(root_directory: Path) -> None:\n with belief_scope(f\"Consolidate archives in {root_directory}\"):\n assert isinstance(root_directory, Path), \"root_directory must be a Path object.\" \n assert root_directory.is_dir(), \"root_directory must be an existing directory.\" \n \n app_logger.info(\"[consolidate_archive_folders][Enter] Consolidating archives in %s\", root_directory) \n # Собираем все директории с архивами \n archive_dirs = [] \n for item in root_directory.iterdir(): \n if item.is_dir(): \n # Проверяем, есть ли в директории ZIP-архивы \n if any(item.glob(\"*.zip\")): \n archive_dirs.append(item) \n # Группируем по слагу (части имени до первого '_') \n slug_groups = {} \n for dir_path in archive_dirs: \n dir_name = dir_path.name \n slug = dir_name.split('_')[0] if '_' in dir_name else dir_name \n if slug not in slug_groups: \n slug_groups[slug] = [] \n slug_groups[slug].append(dir_path) \n # Для каждой группы консолидируем \n for slug, dirs in slug_groups.items(): \n if len(dirs) <= 1: \n continue \n # Создаем целевую директорию \n target_dir = root_directory / slug \n target_dir.mkdir(exist_ok=True) \n app_logger.info(\"[consolidate_archive_folders][State] Consolidating %d directories under %s\", len(dirs), target_dir) \n # Перемещаем содержимое \n for source_dir in dirs: \n if source_dir == target_dir: \n continue \n for item in source_dir.iterdir(): \n dest_item = target_dir / item.name \n try: \n if item.is_dir(): \n shutil.move(str(item), str(dest_item)) \n else: \n shutil.move(str(item), str(dest_item)) \n except Exception as e: \n app_logger.error(\"[consolidate_archive_folders][Failure] Failed to move %s to %s: %s\", item, dest_item, e) \n # Удаляем исходную директорию \n try: \n source_dir.rmdir() \n app_logger.info(\"[consolidate_archive_folders][State] Removed source directory: %s\", source_dir) \n except Exception as e: \n app_logger.error(\"[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s\", source_dir, e) \n# [/DEF:consolidate_archive_folders:Function]\n" + "body": "# [DEF:consolidate_archive_folders:Function]\n# @PURPOSE: Консолидирует директории архивов на основе общего слага в имени.\n# @PRE: root_directory должен быть объектом Path к существующей директории.\n# @POST: Директории с одинаковым префиксом объединены в одну.\n# @THROW: TypeError, ValueError - Если `root_directory` невалиден.\n# @PARAM: root_directory (Path) - Корневая директория для консолидации.\ndef consolidate_archive_folders(root_directory: Path) -> None:\n with belief_scope(f\"Consolidate archives in {root_directory}\"):\n assert isinstance(root_directory, Path), \"root_directory must be a Path object.\" \n assert root_directory.is_dir(), \"root_directory must be an existing directory.\" \n \n log.reason(\"Consolidating archive folders\", payload={\"root_directory\": str(root_directory)}) \n # Собираем все директории с архивами \n archive_dirs = [] \n for item in root_directory.iterdir(): \n if item.is_dir(): \n # Проверяем, есть ли в директории ZIP-архивы \n if any(item.glob(\"*.zip\")): \n archive_dirs.append(item) \n # Группируем по слагу (части имени до первого '_') \n slug_groups = {} \n for dir_path in archive_dirs: \n dir_name = dir_path.name \n slug = dir_name.split('_')[0] if '_' in dir_name else dir_name \n if slug not in slug_groups: \n slug_groups[slug] = [] \n slug_groups[slug].append(dir_path) \n # Для каждой группы консолидируем \n for slug, dirs in slug_groups.items(): \n if len(dirs) <= 1: \n continue \n # Создаем целевую директорию \n target_dir = root_directory / slug \n target_dir.mkdir(exist_ok=True) \n log.reason(\"Consolidating archive directories\", payload={\"slug\": slug, \"dir_count\": len(dirs), \"target\": str(target_dir)}) \n # Перемещаем содержимое \n for source_dir in dirs: \n if source_dir == target_dir: \n continue \n for item in source_dir.iterdir(): \n dest_item = target_dir / item.name \n try: \n if item.is_dir(): \n shutil.move(str(item), str(dest_item)) \n else: \n shutil.move(str(item), str(dest_item)) \n except Exception as e: \n log.explore(\"Failed to move item during consolidation\", payload={\"source\": str(item), \"dest\": str(dest_item)}, error=str(e)) \n # Удаляем исходную директорию \n try: \n source_dir.rmdir() \n log.reason(\"Removed source directory after consolidation\", payload={\"directory\": str(source_dir)}) \n except Exception as e: \n log.explore(\"Failed to remove source directory after consolidation\", payload={\"directory\": str(source_dir)}, error=str(e)) \n# [/DEF:consolidate_archive_folders:Function]\n" }, { "contract_id": "FuzzyMatching", @@ -49869,7 +50625,7 @@ "contract_type": "Module", "file_path": "backend/src/core/utils/network.py", "start_line": 1, - "end_line": 571, + "end_line": 573, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -49921,14 +50677,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:NetworkModule:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: network, http, client, api, requests, session, authentication\n# @PURPOSE: Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.\n# @LAYER: Infra\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\n# @PUBLIC_API: APIClient\n\n# [SECTION: IMPORTS]\nfrom typing import Optional, Dict, Any, List, Union, cast, Tuple\nimport json\nimport io\nfrom pathlib import Path\nimport threading\nimport time\nimport requests\nfrom requests.adapters import HTTPAdapter\nimport urllib3\nfrom urllib3.util.retry import Retry\nfrom ..logger import logger as app_logger, belief_scope\n# [/SECTION]\n\n# [DEF:SupersetAPIError:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Base exception for all Superset API related errors.\nclass SupersetAPIError(Exception):\n # [DEF:__init__:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initializes the exception with a message and context.\n # @PRE: message is a string, context is a dict.\n # @POST: Exception is initialized with context.\n def __init__(self, message: str = \"Superset API error\", **context: Any):\n with belief_scope(\"SupersetAPIError.__init__\"):\n self.context = context\n super().__init__(f\"[API_FAILURE] {message} | Context: {self.context}\")\n # [/DEF:__init__:Function]\n# [/DEF:SupersetAPIError:Class]\n\n# [DEF:AuthenticationError:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Exception raised when authentication fails.\nclass AuthenticationError(SupersetAPIError):\n # [DEF:__init__:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initializes the authentication error.\n # @PRE: message is a string, context is a dict.\n # @POST: AuthenticationError is initialized.\n def __init__(self, message: str = \"Authentication failed\", **context: Any):\n with belief_scope(\"AuthenticationError.__init__\"):\n super().__init__(message, type=\"authentication\", **context)\n # [/DEF:__init__:Function]\n# [/DEF:AuthenticationError:Class]\n\n# [DEF:PermissionDeniedError:Class]\n# @PURPOSE: Exception raised when access is denied.\nclass PermissionDeniedError(AuthenticationError):\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the permission denied error.\n # @PRE: message is a string, context is a dict.\n # @POST: PermissionDeniedError is initialized.\n def __init__(self, message: str = \"Permission denied\", **context: Any):\n with belief_scope(\"PermissionDeniedError.__init__\"):\n super().__init__(message, **context)\n # [/DEF:__init__:Function]\n# [/DEF:PermissionDeniedError:Class]\n\n# [DEF:DashboardNotFoundError:Class]\n# @PURPOSE: Exception raised when a dashboard cannot be found.\nclass DashboardNotFoundError(SupersetAPIError):\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the not found error with resource ID.\n # @PRE: resource_id is provided.\n # @POST: DashboardNotFoundError is initialized.\n def __init__(self, resource_id: Union[int, str], message: str = \"Dashboard not found\", **context: Any):\n with belief_scope(\"DashboardNotFoundError.__init__\"):\n super().__init__(f\"Dashboard '{resource_id}' {message}\", subtype=\"not_found\", resource_id=resource_id, **context)\n # [/DEF:__init__:Function]\n# [/DEF:DashboardNotFoundError:Class]\n\n# [DEF:NetworkError:Class]\n# @PURPOSE: Exception raised when a network level error occurs.\nclass NetworkError(Exception):\n # [DEF:NetworkError.__init__:Function]\n # @PURPOSE: Initializes the network error.\n # @PRE: message is a string.\n # @POST: NetworkError is initialized.\n def __init__(self, message: str = \"Network connection failed\", **context: Any):\n with belief_scope(\"NetworkError.__init__\"):\n self.context = context\n super().__init__(f\"[NETWORK_FAILURE] {message} | Context: {self.context}\")\n # [/DEF:NetworkError.__init__:Function]\n# [/DEF:NetworkError:Class]\n\n\n# [DEF:SupersetAuthCache:Class]\n# @PURPOSE: Process-local cache for Superset access/csrf tokens keyed by environment credentials.\n# @PRE: base_url and username are stable strings.\n# @POST: Cached entries expire automatically by TTL and can be reused across requests.\nclass SupersetAuthCache:\n TTL_SECONDS = 300\n\n _lock = threading.Lock()\n _entries: Dict[Tuple[str, str, bool], Dict[str, Any]] = {}\n\n @classmethod\n def build_key(cls, base_url: str, auth: Optional[Dict[str, Any]], verify_ssl: bool) -> Tuple[str, str, bool]:\n username = \"\"\n if isinstance(auth, dict):\n username = str(auth.get(\"username\") or \"\").strip()\n return (str(base_url or \"\").strip(), username, bool(verify_ssl))\n\n @classmethod\n # [DEF:SupersetAuthCache.get:Function]\n def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]:\n now = time.time()\n with cls._lock:\n payload = cls._entries.get(key)\n if not payload:\n return None\n expires_at = float(payload.get(\"expires_at\") or 0)\n if expires_at <= now:\n cls._entries.pop(key, None)\n return None\n tokens = payload.get(\"tokens\")\n if not isinstance(tokens, dict):\n cls._entries.pop(key, None)\n return None\n return {\n \"access_token\": str(tokens.get(\"access_token\") or \"\"),\n \"csrf_token\": str(tokens.get(\"csrf_token\") or \"\"),\n }\n # [/DEF:SupersetAuthCache.get:Function]\n\n @classmethod\n # [DEF:SupersetAuthCache.set:Function]\n def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None:\n normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1)\n with cls._lock:\n cls._entries[key] = {\n \"tokens\": {\n \"access_token\": str(tokens.get(\"access_token\") or \"\"),\n \"csrf_token\": str(tokens.get(\"csrf_token\") or \"\"),\n },\n \"expires_at\": time.time() + normalized_ttl,\n }\n # [/DEF:SupersetAuthCache.set:Function]\n\n @classmethod\n def invalidate(cls, key: Tuple[str, str, bool]) -> None:\n with cls._lock:\n cls._entries.pop(key, None)\n# [/DEF:SupersetAuthCache:Class]\n\n# [DEF:APIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Synchronous Superset API client with process-local auth token caching.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\nclass APIClient:\n DEFAULT_TIMEOUT = 30\n\n # [DEF:APIClient.__init__:Function]\n # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.\n # @PARAM: config (Dict[str, Any]) - Конфигурация.\n # @PARAM: verify_ssl (bool) - Проверять ли SSL.\n # @PARAM: timeout (int) - Таймаут запросов.\n # @PRE: config must contain 'base_url' and 'auth'.\n # @POST: APIClient instance is initialized with a session.\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n with belief_scope(\"__init__\"):\n app_logger.info(\"[APIClient.__init__][Entry] Initializing APIClient.\")\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self.session = self._init_session()\n self._tokens: Dict[str, str] = {}\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n self._authenticated = False\n app_logger.info(\"[APIClient.__init__][Exit] APIClient initialized.\")\n # [/DEF:APIClient.__init__:Function]\n\n # [DEF:_init_session:Function]\n # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.\n # @PRE: self.request_settings must be initialized.\n # @POST: Returns a configured requests.Session instance.\n # @RETURN: requests.Session - Настроенная сессия.\n def _init_session(self) -> requests.Session:\n with belief_scope(\"_init_session\"):\n session = requests.Session()\n \n # Create a custom adapter that handles TLS issues\n class TLSAdapter(HTTPAdapter):\n def init_poolmanager(self, connections, maxsize, block=False):\n from urllib3.poolmanager import PoolManager\n import ssl\n \n # Create an SSL context that ignores TLSv1 unrecognized name errors\n ctx = ssl.create_default_context()\n ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')\n \n # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification\n # This is safe when verify_ssl is false (we're already not verifying the certificate)\n ctx.check_hostname = False\n \n self.poolmanager = PoolManager(\n num_pools=connections,\n maxsize=maxsize,\n block=block,\n ssl_context=ctx\n )\n \n retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])\n adapter = TLSAdapter(max_retries=retries)\n session.mount('http://', adapter)\n session.mount('https://', adapter)\n \n if not self.request_settings[\"verify_ssl\"]:\n urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n app_logger.warning(\"[_init_session][State] SSL verification disabled.\")\n # When verify_ssl is false, we should also disable hostname verification\n session.verify = False\n else:\n session.verify = True\n \n return session\n # [/DEF:_init_session:Function]\n\n # [DEF:_normalize_base_url:Function]\n # @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.\n # @PRE: raw_url can be empty.\n # @POST: Returns canonical base URL suitable for building API endpoints.\n # @RETURN: str\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:_normalize_base_url:Function]\n\n # [DEF:_build_api_url:Function]\n # @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.\n # @PRE: endpoint is relative path or absolute URL.\n # @POST: Returns full URL without accidental duplicate slashes.\n # @RETURN: str\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:_build_api_url:Function]\n\n # [DEF:APIClient.authenticate:Function]\n # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.\n # @PRE: self.auth and self.base_url must be valid.\n # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.\n # @RETURN: Dict[str, str] - Словарь с токенами.\n # @THROW: AuthenticationError, NetworkError - при ошибках.\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.\n # When a new APIClient instance reuses cached tokens, its fresh requests.Session\n # has no session cookie — causing superset CSRF check to fail with\n # \"CSRF session token is missing\". The fix is to always fetch a fresh CSRF token\n # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie\n # in the new session while avoiding a full POST login.\n # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing\n # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"authenticate\"):\n app_logger.info(\"[authenticate][Enter] Authenticating to %s\", self.base_url)\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this session.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n timeout=self.request_settings[\"timeout\"],\n )\n csrf_response.raise_for_status()\n # Update CSRF token (may have rotated)\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s\", self.base_url)\n except Exception as csrf_err:\n # CSRF refresh failed — likely access_token expired, do full re-auth\n app_logger.warning(\"[authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s\", csrf_err)\n SupersetAuthCache.invalidate(self._auth_cache_key)\n # Fall through to full login below\n self._tokens = {}\n self._authenticated = False\n\n if not self._authenticated:\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n # Log the payload keys and values (masking password)\n masked_auth = {k: (\"******\" if k == \"password\" else v) for k, v in self.auth.items()}\n app_logger.info(f\"[authenticate][Debug] Login URL: {login_url}\")\n app_logger.info(f\"[authenticate][Debug] Auth payload: {masked_auth}\")\n\n response = self.session.post(login_url, json=self.auth, timeout=self.request_settings[\"timeout\"])\n\n if response.status_code != 200:\n app_logger.error(f\"[authenticate][Error] Status: {response.status_code}, Response: {response.text}\")\n\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(csrf_url, headers={\"Authorization\": f\"Bearer {access_token}\"}, timeout=self.request_settings[\"timeout\"])\n csrf_response.raise_for_status()\n\n self._tokens = {\"access_token\": access_token, \"csrf_token\": csrf_response.json()[\"result\"]}\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[authenticate][Exit] Authenticated successfully.\")\n return self._tokens\n except requests.exceptions.HTTPError as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = e.response.status_code if e.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable during authentication (Status {status_code})\", status_code=status_code) from e\n raise AuthenticationError(f\"Authentication failed: {e}\") from e\n except (requests.exceptions.RequestException, KeyError) as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {e}\") from e\n\n self._authenticated = True\n app_logger.info(\"[authenticate][Exit] Authenticated successfully (from cache).\")\n return self._tokens\n # [/DEF:APIClient.authenticate:Function]\n\n @property\n # [DEF:headers:Function]\n # @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.\n # @PRE: APIClient is initialized and authenticated or can be authenticated.\n # @POST: Returns headers including auth tokens.\n def headers(self) -> Dict[str, str]:\n if not self._authenticated:\n self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\"\n }\n # [/DEF:headers:Function]\n\n # [DEF:request:Function]\n # @PURPOSE: Выполняет универсальный HTTP-запрос к API.\n # @PARAM: method (str) - HTTP метод.\n # @PARAM: endpoint (str) - API эндпоинт.\n # @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.\n # @PARAM: raw_response (bool) - Возвращать ли сырой ответ.\n # @PRE: method and endpoint must be strings.\n # @POST: Returns response content or raw Response object.\n # @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.\n # @THROW: SupersetAPIError, NetworkError и их подклассы.\n def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n if headers:\n _headers.update(headers)\n \n try:\n response = self.session.request(method, full_url, headers=_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except requests.exceptions.HTTPError as e:\n if e.response is not None and e.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(e, endpoint)\n except requests.exceptions.RequestException as e:\n self._handle_network_error(e, full_url)\n # [/DEF:request:Function]\n\n # [DEF:_handle_http_error:Function]\n # @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.\n # @PARAM: e (requests.exceptions.HTTPError) - Ошибка.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PRE: e must be a valid HTTPError with a response.\n # @POST: Raises a specific SupersetAPIError or subclass.\n def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str):\n with belief_scope(\"_handle_http_error\"):\n status_code = e.response.status_code\n if status_code == 502 or status_code == 503 or status_code == 504:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from e\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from e\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from e\n if status_code == 403:\n raise PermissionDeniedError() from e\n if status_code == 401:\n raise AuthenticationError() from e\n raise SupersetAPIError(f\"API Error {status_code}: {e.response.text}\") from e\n # [/DEF:_handle_http_error:Function]\n\n # [DEF:_is_dashboard_endpoint:Function]\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @PRE: endpoint may be relative or absolute.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:_is_dashboard_endpoint:Function]\n\n # [DEF:_handle_network_error:Function]\n # @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.\n # @PARAM: e (requests.exceptions.RequestException) - Ошибка.\n # @PARAM: url (str) - URL.\n # @PRE: e must be a RequestException.\n # @POST: Raises a NetworkError.\n def _handle_network_error(self, e: requests.exceptions.RequestException, url: str):\n with belief_scope(\"_handle_network_error\"):\n if isinstance(e, requests.exceptions.Timeout):\n msg = \"Request timeout\"\n elif isinstance(e, requests.exceptions.ConnectionError):\n msg = \"Connection error\"\n else:\n msg = f\"Unknown network error: {e}\"\n raise NetworkError(msg, url=url) from e\n # [/DEF:_handle_network_error:Function]\n\n # [DEF:upload_file:Function]\n # @PURPOSE: Загружает файл на сервер через multipart/form-data.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: file_info (Dict[str, Any]) - Информация о файле.\n # @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: file_info must contain 'file_obj' and 'file_name'.\n # @POST: File is uploaded and response returned.\n # @RETURN: Ответ API в виде словаря.\n # @THROW: SupersetAPIError, NetworkError, TypeError.\n def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:\n with belief_scope(\"upload_file\"):\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n _headers.pop('Content-Type', None)\n \n \n file_obj, file_name, form_field = file_info.get(\"file_obj\"), file_info.get(\"file_name\"), file_info.get(\"form_field\", \"file\")\n \n files_payload = {}\n if isinstance(file_obj, (str, Path)):\n with open(file_obj, 'rb') as f:\n files_payload = {form_field: (file_name, f.read(), 'application/x-zip-compressed')}\n elif isinstance(file_obj, io.BytesIO):\n files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}\n else:\n raise TypeError(f\"Unsupported file_obj type: {type(file_obj)}\")\n \n return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)\n # [/DEF:upload_file:Function]\n\n # [DEF:_perform_upload:Function]\n # @PURPOSE: (Helper) Выполняет POST запрос с файлом.\n # @PARAM: url (str) - URL.\n # @PARAM: files (Dict) - Файлы.\n # @PARAM: data (Optional[Dict]) - Данные.\n # @PARAM: headers (Dict) - Заголовки.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: url, files, and headers must be provided.\n # @POST: POST request is performed and JSON response returned.\n # @RETURN: Dict - Ответ.\n def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:\n with belief_scope(\"_perform_upload\"):\n try:\n response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings[\"timeout\"])\n response.raise_for_status()\n if response.status_code == 200:\n try:\n return response.json()\n except Exception as json_e:\n app_logger.debug(f\"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...\")\n raise SupersetAPIError(f\"API error during upload: Response is not valid JSON: {json_e}\") from json_e\n return response.json()\n except requests.exceptions.HTTPError as e:\n raise SupersetAPIError(f\"API error during upload: {e.response.text}\") from e\n except requests.exceptions.RequestException as e:\n raise NetworkError(f\"Network error during upload: {e}\", url=url) from e\n # [/DEF:_perform_upload:Function]\n\n # [DEF:fetch_paginated_count:Function]\n # @PURPOSE: Получает общее количество элементов для пагинации.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: query_params (Dict) - Параметры запроса.\n # @PARAM: count_field (str) - Поле с количеством.\n # @PRE: query_params must be a dictionary.\n # @POST: Returns total count of items.\n # @RETURN: int - Количество.\n def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = \"count\") -> int:\n with belief_scope(\"fetch_paginated_count\"):\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query_params)}))\n return response_json.get(count_field, 0)\n # [/DEF:fetch_paginated_count:Function]\n\n # [DEF:fetch_paginated_data:Function]\n # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.\n # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.\n # @POST: Returns all items across all pages.\n # @RETURN: List[Any] - Список данных.\n def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:\n with belief_scope(\"fetch_paginated_data\"):\n base_query = pagination_options[\"base_query\"]\n total_count = pagination_options.get(\"total_count\")\n \n results_field = pagination_options[\"results_field\"]\n count_field = pagination_options.get(\"count_field\", \"count\")\n page_size = base_query.get('page_size', 1000)\n assert page_size > 0, \"'page_size' must be a positive number.\"\n \n results = []\n page = 0\n \n # Fetch first page to get data and total count if not provided\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n \n first_page_results = response_json.get(results_field, [])\n results.extend(first_page_results)\n \n if total_count is None:\n total_count = response_json.get(count_field, len(first_page_results))\n app_logger.debug(f\"[fetch_paginated_data][State] Total count resolved from first page: {total_count}\")\n\n # Fetch remaining pages\n total_pages = (total_count + page_size - 1) // page_size\n for page in range(1, total_pages):\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n results.extend(response_json.get(results_field, []))\n \n return results\n # [/DEF:fetch_paginated_data:Function]\n\n# [/DEF:APIClient:Class]\n\n# [/DEF:NetworkModule:Module]\n" + "body": "# [DEF:NetworkModule:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: network, http, client, api, requests, session, authentication\n# @PURPOSE: Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.\n# @LAYER: Infra\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\n# @PUBLIC_API: APIClient\n\n# [SECTION: IMPORTS]\nfrom typing import Optional, Dict, Any, List, Union, cast, Tuple\nimport json\nimport io\nfrom pathlib import Path\nimport threading\nimport time\nimport requests\nfrom requests.adapters import HTTPAdapter\nimport urllib3\nfrom urllib3.util.retry import Retry\nfrom ..logger import logger as app_logger, belief_scope\nfrom ..cot_logger import MarkerLogger\n# [/SECTION]\n\nlog = MarkerLogger(\"Network\")\n\n# [DEF:SupersetAPIError:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Base exception for all Superset API related errors.\nclass SupersetAPIError(Exception):\n # [DEF:__init__:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initializes the exception with a message and context.\n # @PRE: message is a string, context is a dict.\n # @POST: Exception is initialized with context.\n def __init__(self, message: str = \"Superset API error\", **context: Any):\n with belief_scope(\"SupersetAPIError.__init__\"):\n self.context = context\n super().__init__(f\"[API_FAILURE] {message} | Context: {self.context}\")\n # [/DEF:__init__:Function]\n# [/DEF:SupersetAPIError:Class]\n\n# [DEF:AuthenticationError:Class]\n# @COMPLEXITY: 1\n# @PURPOSE: Exception raised when authentication fails.\nclass AuthenticationError(SupersetAPIError):\n # [DEF:__init__:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initializes the authentication error.\n # @PRE: message is a string, context is a dict.\n # @POST: AuthenticationError is initialized.\n def __init__(self, message: str = \"Authentication failed\", **context: Any):\n with belief_scope(\"AuthenticationError.__init__\"):\n super().__init__(message, type=\"authentication\", **context)\n # [/DEF:__init__:Function]\n# [/DEF:AuthenticationError:Class]\n\n# [DEF:PermissionDeniedError:Class]\n# @PURPOSE: Exception raised when access is denied.\nclass PermissionDeniedError(AuthenticationError):\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the permission denied error.\n # @PRE: message is a string, context is a dict.\n # @POST: PermissionDeniedError is initialized.\n def __init__(self, message: str = \"Permission denied\", **context: Any):\n with belief_scope(\"PermissionDeniedError.__init__\"):\n super().__init__(message, **context)\n # [/DEF:__init__:Function]\n# [/DEF:PermissionDeniedError:Class]\n\n# [DEF:DashboardNotFoundError:Class]\n# @PURPOSE: Exception raised when a dashboard cannot be found.\nclass DashboardNotFoundError(SupersetAPIError):\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the not found error with resource ID.\n # @PRE: resource_id is provided.\n # @POST: DashboardNotFoundError is initialized.\n def __init__(self, resource_id: Union[int, str], message: str = \"Dashboard not found\", **context: Any):\n with belief_scope(\"DashboardNotFoundError.__init__\"):\n super().__init__(f\"Dashboard '{resource_id}' {message}\", subtype=\"not_found\", resource_id=resource_id, **context)\n # [/DEF:__init__:Function]\n# [/DEF:DashboardNotFoundError:Class]\n\n# [DEF:NetworkError:Class]\n# @PURPOSE: Exception raised when a network level error occurs.\nclass NetworkError(Exception):\n # [DEF:NetworkError.__init__:Function]\n # @PURPOSE: Initializes the network error.\n # @PRE: message is a string.\n # @POST: NetworkError is initialized.\n def __init__(self, message: str = \"Network connection failed\", **context: Any):\n with belief_scope(\"NetworkError.__init__\"):\n self.context = context\n super().__init__(f\"[NETWORK_FAILURE] {message} | Context: {self.context}\")\n # [/DEF:NetworkError.__init__:Function]\n# [/DEF:NetworkError:Class]\n\n\n# [DEF:SupersetAuthCache:Class]\n# @PURPOSE: Process-local cache for Superset access/csrf tokens keyed by environment credentials.\n# @PRE: base_url and username are stable strings.\n# @POST: Cached entries expire automatically by TTL and can be reused across requests.\nclass SupersetAuthCache:\n TTL_SECONDS = 300\n\n _lock = threading.Lock()\n _entries: Dict[Tuple[str, str, bool], Dict[str, Any]] = {}\n\n @classmethod\n def build_key(cls, base_url: str, auth: Optional[Dict[str, Any]], verify_ssl: bool) -> Tuple[str, str, bool]:\n username = \"\"\n if isinstance(auth, dict):\n username = str(auth.get(\"username\") or \"\").strip()\n return (str(base_url or \"\").strip(), username, bool(verify_ssl))\n\n @classmethod\n # [DEF:SupersetAuthCache.get:Function]\n def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]:\n now = time.time()\n with cls._lock:\n payload = cls._entries.get(key)\n if not payload:\n return None\n expires_at = float(payload.get(\"expires_at\") or 0)\n if expires_at <= now:\n cls._entries.pop(key, None)\n return None\n tokens = payload.get(\"tokens\")\n if not isinstance(tokens, dict):\n cls._entries.pop(key, None)\n return None\n return {\n \"access_token\": str(tokens.get(\"access_token\") or \"\"),\n \"csrf_token\": str(tokens.get(\"csrf_token\") or \"\"),\n }\n # [/DEF:SupersetAuthCache.get:Function]\n\n @classmethod\n # [DEF:SupersetAuthCache.set:Function]\n def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None:\n normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1)\n with cls._lock:\n cls._entries[key] = {\n \"tokens\": {\n \"access_token\": str(tokens.get(\"access_token\") or \"\"),\n \"csrf_token\": str(tokens.get(\"csrf_token\") or \"\"),\n },\n \"expires_at\": time.time() + normalized_ttl,\n }\n # [/DEF:SupersetAuthCache.set:Function]\n\n @classmethod\n def invalidate(cls, key: Tuple[str, str, bool]) -> None:\n with cls._lock:\n cls._entries.pop(key, None)\n# [/DEF:SupersetAuthCache:Class]\n\n# [DEF:APIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Synchronous Superset API client with process-local auth token caching.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\nclass APIClient:\n DEFAULT_TIMEOUT = 30\n\n # [DEF:APIClient.__init__:Function]\n # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.\n # @PARAM: config (Dict[str, Any]) - Конфигурация.\n # @PARAM: verify_ssl (bool) - Проверять ли SSL.\n # @PARAM: timeout (int) - Таймаут запросов.\n # @PRE: config must contain 'base_url' and 'auth'.\n # @POST: APIClient instance is initialized with a session.\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n with belief_scope(\"__init__\"):\n log.reason(\"Initializing APIClient\")\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self.session = self._init_session()\n self._tokens: Dict[str, str] = {}\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n self._authenticated = False\n log.reflect(\"APIClient initialized\")\n # [/DEF:APIClient.__init__:Function]\n\n # [DEF:_init_session:Function]\n # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.\n # @PRE: self.request_settings must be initialized.\n # @POST: Returns a configured requests.Session instance.\n # @RETURN: requests.Session - Настроенная сессия.\n def _init_session(self) -> requests.Session:\n with belief_scope(\"_init_session\"):\n session = requests.Session()\n \n # Create a custom adapter that handles TLS issues\n class TLSAdapter(HTTPAdapter):\n def init_poolmanager(self, connections, maxsize, block=False):\n from urllib3.poolmanager import PoolManager\n import ssl\n \n # Create an SSL context that ignores TLSv1 unrecognized name errors\n ctx = ssl.create_default_context()\n ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')\n \n # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification\n # This is safe when verify_ssl is false (we're already not verifying the certificate)\n ctx.check_hostname = False\n \n self.poolmanager = PoolManager(\n num_pools=connections,\n maxsize=maxsize,\n block=block,\n ssl_context=ctx\n )\n \n retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])\n adapter = TLSAdapter(max_retries=retries)\n session.mount('http://', adapter)\n session.mount('https://', adapter)\n \n if not self.request_settings[\"verify_ssl\"]:\n urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n log.reflect(\"SSL verification disabled for session\", payload={\"base_url\": self.base_url})\n # When verify_ssl is false, we should also disable hostname verification\n session.verify = False\n else:\n session.verify = True\n \n return session\n # [/DEF:_init_session:Function]\n\n # [DEF:_normalize_base_url:Function]\n # @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.\n # @PRE: raw_url can be empty.\n # @POST: Returns canonical base URL suitable for building API endpoints.\n # @RETURN: str\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:_normalize_base_url:Function]\n\n # [DEF:_build_api_url:Function]\n # @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.\n # @PRE: endpoint is relative path or absolute URL.\n # @POST: Returns full URL without accidental duplicate slashes.\n # @RETURN: str\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:_build_api_url:Function]\n\n # [DEF:APIClient.authenticate:Function]\n # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.\n # @PRE: self.auth and self.base_url must be valid.\n # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.\n # @RETURN: Dict[str, str] - Словарь с токенами.\n # @THROW: AuthenticationError, NetworkError - при ошибках.\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.\n # When a new APIClient instance reuses cached tokens, its fresh requests.Session\n # has no session cookie — causing superset CSRF check to fail with\n # \"CSRF session token is missing\". The fix is to always fetch a fresh CSRF token\n # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie\n # in the new session while avoiding a full POST login.\n # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing\n # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"authenticate\"):\n log.reason(\"Authenticating to Superset\", payload={\"base_url\": self.base_url})\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this session.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n timeout=self.request_settings[\"timeout\"],\n )\n csrf_response.raise_for_status()\n # Update CSRF token (may have rotated)\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reason(\"Reused cached tokens + refreshed CSRF token\", payload={\"base_url\": self.base_url})\n except Exception as csrf_err:\n # CSRF refresh failed — likely access_token expired, do full re-auth\n log.explore(\"CSRF refresh failed, performing full re-auth\", error=str(csrf_err))\n SupersetAuthCache.invalidate(self._auth_cache_key)\n # Fall through to full login below\n self._tokens = {}\n self._authenticated = False\n\n if not self._authenticated:\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n # Log the payload keys and values (masking password)\n masked_auth = {k: (\"******\" if k == \"password\" else v) for k, v in self.auth.items()}\n log.reason(\"Performing full Superset login\", payload={\"login_url\": login_url, \"auth_payload\": masked_auth})\n\n response = self.session.post(login_url, json=self.auth, timeout=self.request_settings[\"timeout\"])\n\n if response.status_code != 200:\n log.explore(\"Superset login returned non-200 status\", payload={\"status_code\": response.status_code, \"response\": response.text}, error=f\"HTTP {response.status_code}: {response.text[:200]}\")\n\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(csrf_url, headers={\"Authorization\": f\"Bearer {access_token}\"}, timeout=self.request_settings[\"timeout\"])\n csrf_response.raise_for_status()\n\n self._tokens = {\"access_token\": access_token, \"csrf_token\": csrf_response.json()[\"result\"]}\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reflect(\"Authenticated successfully\")\n return self._tokens\n except requests.exceptions.HTTPError as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = e.response.status_code if e.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable during authentication (Status {status_code})\", status_code=status_code) from e\n raise AuthenticationError(f\"Authentication failed: {e}\") from e\n except (requests.exceptions.RequestException, KeyError) as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {e}\") from e\n\n self._authenticated = True\n log.reflect(\"Authenticated successfully (from cached tokens)\")\n return self._tokens\n # [/DEF:APIClient.authenticate:Function]\n\n @property\n # [DEF:headers:Function]\n # @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.\n # @PRE: APIClient is initialized and authenticated or can be authenticated.\n # @POST: Returns headers including auth tokens.\n def headers(self) -> Dict[str, str]:\n if not self._authenticated:\n self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\"\n }\n # [/DEF:headers:Function]\n\n # [DEF:request:Function]\n # @PURPOSE: Выполняет универсальный HTTP-запрос к API.\n # @PARAM: method (str) - HTTP метод.\n # @PARAM: endpoint (str) - API эндпоинт.\n # @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.\n # @PARAM: raw_response (bool) - Возвращать ли сырой ответ.\n # @PRE: method and endpoint must be strings.\n # @POST: Returns response content or raw Response object.\n # @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.\n # @THROW: SupersetAPIError, NetworkError и их подклассы.\n def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n if headers:\n _headers.update(headers)\n \n try:\n response = self.session.request(method, full_url, headers=_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except requests.exceptions.HTTPError as e:\n if e.response is not None and e.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(e, endpoint)\n except requests.exceptions.RequestException as e:\n self._handle_network_error(e, full_url)\n # [/DEF:request:Function]\n\n # [DEF:_handle_http_error:Function]\n # @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.\n # @PARAM: e (requests.exceptions.HTTPError) - Ошибка.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PRE: e must be a valid HTTPError with a response.\n # @POST: Raises a specific SupersetAPIError or subclass.\n def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str):\n with belief_scope(\"_handle_http_error\"):\n status_code = e.response.status_code\n if status_code == 502 or status_code == 503 or status_code == 504:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from e\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from e\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from e\n if status_code == 403:\n raise PermissionDeniedError() from e\n if status_code == 401:\n raise AuthenticationError() from e\n raise SupersetAPIError(f\"API Error {status_code}: {e.response.text}\") from e\n # [/DEF:_handle_http_error:Function]\n\n # [DEF:_is_dashboard_endpoint:Function]\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @PRE: endpoint may be relative or absolute.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:_is_dashboard_endpoint:Function]\n\n # [DEF:_handle_network_error:Function]\n # @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.\n # @PARAM: e (requests.exceptions.RequestException) - Ошибка.\n # @PARAM: url (str) - URL.\n # @PRE: e must be a RequestException.\n # @POST: Raises a NetworkError.\n def _handle_network_error(self, e: requests.exceptions.RequestException, url: str):\n with belief_scope(\"_handle_network_error\"):\n if isinstance(e, requests.exceptions.Timeout):\n msg = \"Request timeout\"\n elif isinstance(e, requests.exceptions.ConnectionError):\n msg = \"Connection error\"\n else:\n msg = f\"Unknown network error: {e}\"\n raise NetworkError(msg, url=url) from e\n # [/DEF:_handle_network_error:Function]\n\n # [DEF:upload_file:Function]\n # @PURPOSE: Загружает файл на сервер через multipart/form-data.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: file_info (Dict[str, Any]) - Информация о файле.\n # @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: file_info must contain 'file_obj' and 'file_name'.\n # @POST: File is uploaded and response returned.\n # @RETURN: Ответ API в виде словаря.\n # @THROW: SupersetAPIError, NetworkError, TypeError.\n def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:\n with belief_scope(\"upload_file\"):\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n _headers.pop('Content-Type', None)\n \n \n file_obj, file_name, form_field = file_info.get(\"file_obj\"), file_info.get(\"file_name\"), file_info.get(\"form_field\", \"file\")\n \n files_payload = {}\n if isinstance(file_obj, (str, Path)):\n with open(file_obj, 'rb') as f:\n files_payload = {form_field: (file_name, f.read(), 'application/x-zip-compressed')}\n elif isinstance(file_obj, io.BytesIO):\n files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}\n else:\n raise TypeError(f\"Unsupported file_obj type: {type(file_obj)}\")\n \n return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)\n # [/DEF:upload_file:Function]\n\n # [DEF:_perform_upload:Function]\n # @PURPOSE: (Helper) Выполняет POST запрос с файлом.\n # @PARAM: url (str) - URL.\n # @PARAM: files (Dict) - Файлы.\n # @PARAM: data (Optional[Dict]) - Данные.\n # @PARAM: headers (Dict) - Заголовки.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: url, files, and headers must be provided.\n # @POST: POST request is performed and JSON response returned.\n # @RETURN: Dict - Ответ.\n def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:\n with belief_scope(\"_perform_upload\"):\n try:\n response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings[\"timeout\"])\n response.raise_for_status()\n if response.status_code == 200:\n try:\n return response.json()\n except Exception as json_e:\n log.explore(\"Upload response not valid JSON\", payload={\"preview\": response.text[:200]}, error=str(json_e))\n raise SupersetAPIError(f\"API error during upload: Response is not valid JSON: {json_e}\") from json_e\n return response.json()\n except requests.exceptions.HTTPError as e:\n raise SupersetAPIError(f\"API error during upload: {e.response.text}\") from e\n except requests.exceptions.RequestException as e:\n raise NetworkError(f\"Network error during upload: {e}\", url=url) from e\n # [/DEF:_perform_upload:Function]\n\n # [DEF:fetch_paginated_count:Function]\n # @PURPOSE: Получает общее количество элементов для пагинации.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: query_params (Dict) - Параметры запроса.\n # @PARAM: count_field (str) - Поле с количеством.\n # @PRE: query_params must be a dictionary.\n # @POST: Returns total count of items.\n # @RETURN: int - Количество.\n def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = \"count\") -> int:\n with belief_scope(\"fetch_paginated_count\"):\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query_params)}))\n return response_json.get(count_field, 0)\n # [/DEF:fetch_paginated_count:Function]\n\n # [DEF:fetch_paginated_data:Function]\n # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.\n # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.\n # @POST: Returns all items across all pages.\n # @RETURN: List[Any] - Список данных.\n def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:\n with belief_scope(\"fetch_paginated_data\"):\n base_query = pagination_options[\"base_query\"]\n total_count = pagination_options.get(\"total_count\")\n \n results_field = pagination_options[\"results_field\"]\n count_field = pagination_options.get(\"count_field\", \"count\")\n page_size = base_query.get('page_size', 1000)\n assert page_size > 0, \"'page_size' must be a positive number.\"\n \n results = []\n page = 0\n \n # Fetch first page to get data and total count if not provided\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n \n first_page_results = response_json.get(results_field, [])\n results.extend(first_page_results)\n \n if total_count is None:\n total_count = response_json.get(count_field, len(first_page_results))\n log.reason(\"Total count resolved from first page\", payload={\"total_count\": total_count})\n\n # Fetch remaining pages\n total_pages = (total_count + page_size - 1) // page_size\n for page in range(1, total_pages):\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n results.extend(response_json.get(results_field, []))\n \n return results\n # [/DEF:fetch_paginated_data:Function]\n\n# [/DEF:APIClient:Class]\n\n# [/DEF:NetworkModule:Module]\n" }, { "contract_id": "SupersetAPIError", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 24, - "end_line": 38, + "start_line": 27, + "end_line": 41, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49944,8 +50700,8 @@ "contract_id": "AuthenticationError", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 40, - "end_line": 53, + "start_line": 43, + "end_line": 56, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49961,8 +50717,8 @@ "contract_id": "PermissionDeniedError", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 55, - "end_line": 66, + "start_line": 58, + "end_line": 69, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49977,8 +50733,8 @@ "contract_id": "DashboardNotFoundError", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 68, - "end_line": 79, + "start_line": 71, + "end_line": 82, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -49993,8 +50749,8 @@ "contract_id": "NetworkError", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 81, - "end_line": 93, + "start_line": 84, + "end_line": 96, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50009,8 +50765,8 @@ "contract_id": "NetworkError.__init__", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 84, - "end_line": 92, + "start_line": 87, + "end_line": 95, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50046,8 +50802,8 @@ "contract_id": "SupersetAuthCache", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 96, - "end_line": 153, + "start_line": 99, + "end_line": 156, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50083,8 +50839,8 @@ "contract_id": "SupersetAuthCache.get", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 114, - "end_line": 133, + "start_line": 117, + "end_line": 136, "tier": "TIER_1", "complexity": 1, "metadata": {}, @@ -50107,8 +50863,8 @@ "contract_id": "SupersetAuthCache.set", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 136, - "end_line": 147, + "start_line": 139, + "end_line": 150, "tier": "TIER_1", "complexity": 1, "metadata": {}, @@ -50131,8 +50887,8 @@ "contract_id": "APIClient", "contract_type": "Class", "file_path": "backend/src/core/utils/network.py", - "start_line": 155, - "end_line": 569, + "start_line": 158, + "end_line": 571, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -50190,14 +50946,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:APIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Synchronous Superset API client with process-local auth token caching.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\nclass APIClient:\n DEFAULT_TIMEOUT = 30\n\n # [DEF:APIClient.__init__:Function]\n # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.\n # @PARAM: config (Dict[str, Any]) - Конфигурация.\n # @PARAM: verify_ssl (bool) - Проверять ли SSL.\n # @PARAM: timeout (int) - Таймаут запросов.\n # @PRE: config must contain 'base_url' and 'auth'.\n # @POST: APIClient instance is initialized with a session.\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n with belief_scope(\"__init__\"):\n app_logger.info(\"[APIClient.__init__][Entry] Initializing APIClient.\")\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self.session = self._init_session()\n self._tokens: Dict[str, str] = {}\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n self._authenticated = False\n app_logger.info(\"[APIClient.__init__][Exit] APIClient initialized.\")\n # [/DEF:APIClient.__init__:Function]\n\n # [DEF:_init_session:Function]\n # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.\n # @PRE: self.request_settings must be initialized.\n # @POST: Returns a configured requests.Session instance.\n # @RETURN: requests.Session - Настроенная сессия.\n def _init_session(self) -> requests.Session:\n with belief_scope(\"_init_session\"):\n session = requests.Session()\n \n # Create a custom adapter that handles TLS issues\n class TLSAdapter(HTTPAdapter):\n def init_poolmanager(self, connections, maxsize, block=False):\n from urllib3.poolmanager import PoolManager\n import ssl\n \n # Create an SSL context that ignores TLSv1 unrecognized name errors\n ctx = ssl.create_default_context()\n ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')\n \n # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification\n # This is safe when verify_ssl is false (we're already not verifying the certificate)\n ctx.check_hostname = False\n \n self.poolmanager = PoolManager(\n num_pools=connections,\n maxsize=maxsize,\n block=block,\n ssl_context=ctx\n )\n \n retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])\n adapter = TLSAdapter(max_retries=retries)\n session.mount('http://', adapter)\n session.mount('https://', adapter)\n \n if not self.request_settings[\"verify_ssl\"]:\n urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n app_logger.warning(\"[_init_session][State] SSL verification disabled.\")\n # When verify_ssl is false, we should also disable hostname verification\n session.verify = False\n else:\n session.verify = True\n \n return session\n # [/DEF:_init_session:Function]\n\n # [DEF:_normalize_base_url:Function]\n # @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.\n # @PRE: raw_url can be empty.\n # @POST: Returns canonical base URL suitable for building API endpoints.\n # @RETURN: str\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:_normalize_base_url:Function]\n\n # [DEF:_build_api_url:Function]\n # @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.\n # @PRE: endpoint is relative path or absolute URL.\n # @POST: Returns full URL without accidental duplicate slashes.\n # @RETURN: str\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:_build_api_url:Function]\n\n # [DEF:APIClient.authenticate:Function]\n # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.\n # @PRE: self.auth and self.base_url must be valid.\n # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.\n # @RETURN: Dict[str, str] - Словарь с токенами.\n # @THROW: AuthenticationError, NetworkError - при ошибках.\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.\n # When a new APIClient instance reuses cached tokens, its fresh requests.Session\n # has no session cookie — causing superset CSRF check to fail with\n # \"CSRF session token is missing\". The fix is to always fetch a fresh CSRF token\n # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie\n # in the new session while avoiding a full POST login.\n # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing\n # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"authenticate\"):\n app_logger.info(\"[authenticate][Enter] Authenticating to %s\", self.base_url)\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this session.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n timeout=self.request_settings[\"timeout\"],\n )\n csrf_response.raise_for_status()\n # Update CSRF token (may have rotated)\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s\", self.base_url)\n except Exception as csrf_err:\n # CSRF refresh failed — likely access_token expired, do full re-auth\n app_logger.warning(\"[authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s\", csrf_err)\n SupersetAuthCache.invalidate(self._auth_cache_key)\n # Fall through to full login below\n self._tokens = {}\n self._authenticated = False\n\n if not self._authenticated:\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n # Log the payload keys and values (masking password)\n masked_auth = {k: (\"******\" if k == \"password\" else v) for k, v in self.auth.items()}\n app_logger.info(f\"[authenticate][Debug] Login URL: {login_url}\")\n app_logger.info(f\"[authenticate][Debug] Auth payload: {masked_auth}\")\n\n response = self.session.post(login_url, json=self.auth, timeout=self.request_settings[\"timeout\"])\n\n if response.status_code != 200:\n app_logger.error(f\"[authenticate][Error] Status: {response.status_code}, Response: {response.text}\")\n\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(csrf_url, headers={\"Authorization\": f\"Bearer {access_token}\"}, timeout=self.request_settings[\"timeout\"])\n csrf_response.raise_for_status()\n\n self._tokens = {\"access_token\": access_token, \"csrf_token\": csrf_response.json()[\"result\"]}\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[authenticate][Exit] Authenticated successfully.\")\n return self._tokens\n except requests.exceptions.HTTPError as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = e.response.status_code if e.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable during authentication (Status {status_code})\", status_code=status_code) from e\n raise AuthenticationError(f\"Authentication failed: {e}\") from e\n except (requests.exceptions.RequestException, KeyError) as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {e}\") from e\n\n self._authenticated = True\n app_logger.info(\"[authenticate][Exit] Authenticated successfully (from cache).\")\n return self._tokens\n # [/DEF:APIClient.authenticate:Function]\n\n @property\n # [DEF:headers:Function]\n # @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.\n # @PRE: APIClient is initialized and authenticated or can be authenticated.\n # @POST: Returns headers including auth tokens.\n def headers(self) -> Dict[str, str]:\n if not self._authenticated:\n self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\"\n }\n # [/DEF:headers:Function]\n\n # [DEF:request:Function]\n # @PURPOSE: Выполняет универсальный HTTP-запрос к API.\n # @PARAM: method (str) - HTTP метод.\n # @PARAM: endpoint (str) - API эндпоинт.\n # @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.\n # @PARAM: raw_response (bool) - Возвращать ли сырой ответ.\n # @PRE: method and endpoint must be strings.\n # @POST: Returns response content or raw Response object.\n # @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.\n # @THROW: SupersetAPIError, NetworkError и их подклассы.\n def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n if headers:\n _headers.update(headers)\n \n try:\n response = self.session.request(method, full_url, headers=_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except requests.exceptions.HTTPError as e:\n if e.response is not None and e.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(e, endpoint)\n except requests.exceptions.RequestException as e:\n self._handle_network_error(e, full_url)\n # [/DEF:request:Function]\n\n # [DEF:_handle_http_error:Function]\n # @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.\n # @PARAM: e (requests.exceptions.HTTPError) - Ошибка.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PRE: e must be a valid HTTPError with a response.\n # @POST: Raises a specific SupersetAPIError or subclass.\n def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str):\n with belief_scope(\"_handle_http_error\"):\n status_code = e.response.status_code\n if status_code == 502 or status_code == 503 or status_code == 504:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from e\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from e\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from e\n if status_code == 403:\n raise PermissionDeniedError() from e\n if status_code == 401:\n raise AuthenticationError() from e\n raise SupersetAPIError(f\"API Error {status_code}: {e.response.text}\") from e\n # [/DEF:_handle_http_error:Function]\n\n # [DEF:_is_dashboard_endpoint:Function]\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @PRE: endpoint may be relative or absolute.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:_is_dashboard_endpoint:Function]\n\n # [DEF:_handle_network_error:Function]\n # @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.\n # @PARAM: e (requests.exceptions.RequestException) - Ошибка.\n # @PARAM: url (str) - URL.\n # @PRE: e must be a RequestException.\n # @POST: Raises a NetworkError.\n def _handle_network_error(self, e: requests.exceptions.RequestException, url: str):\n with belief_scope(\"_handle_network_error\"):\n if isinstance(e, requests.exceptions.Timeout):\n msg = \"Request timeout\"\n elif isinstance(e, requests.exceptions.ConnectionError):\n msg = \"Connection error\"\n else:\n msg = f\"Unknown network error: {e}\"\n raise NetworkError(msg, url=url) from e\n # [/DEF:_handle_network_error:Function]\n\n # [DEF:upload_file:Function]\n # @PURPOSE: Загружает файл на сервер через multipart/form-data.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: file_info (Dict[str, Any]) - Информация о файле.\n # @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: file_info must contain 'file_obj' and 'file_name'.\n # @POST: File is uploaded and response returned.\n # @RETURN: Ответ API в виде словаря.\n # @THROW: SupersetAPIError, NetworkError, TypeError.\n def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:\n with belief_scope(\"upload_file\"):\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n _headers.pop('Content-Type', None)\n \n \n file_obj, file_name, form_field = file_info.get(\"file_obj\"), file_info.get(\"file_name\"), file_info.get(\"form_field\", \"file\")\n \n files_payload = {}\n if isinstance(file_obj, (str, Path)):\n with open(file_obj, 'rb') as f:\n files_payload = {form_field: (file_name, f.read(), 'application/x-zip-compressed')}\n elif isinstance(file_obj, io.BytesIO):\n files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}\n else:\n raise TypeError(f\"Unsupported file_obj type: {type(file_obj)}\")\n \n return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)\n # [/DEF:upload_file:Function]\n\n # [DEF:_perform_upload:Function]\n # @PURPOSE: (Helper) Выполняет POST запрос с файлом.\n # @PARAM: url (str) - URL.\n # @PARAM: files (Dict) - Файлы.\n # @PARAM: data (Optional[Dict]) - Данные.\n # @PARAM: headers (Dict) - Заголовки.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: url, files, and headers must be provided.\n # @POST: POST request is performed and JSON response returned.\n # @RETURN: Dict - Ответ.\n def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:\n with belief_scope(\"_perform_upload\"):\n try:\n response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings[\"timeout\"])\n response.raise_for_status()\n if response.status_code == 200:\n try:\n return response.json()\n except Exception as json_e:\n app_logger.debug(f\"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...\")\n raise SupersetAPIError(f\"API error during upload: Response is not valid JSON: {json_e}\") from json_e\n return response.json()\n except requests.exceptions.HTTPError as e:\n raise SupersetAPIError(f\"API error during upload: {e.response.text}\") from e\n except requests.exceptions.RequestException as e:\n raise NetworkError(f\"Network error during upload: {e}\", url=url) from e\n # [/DEF:_perform_upload:Function]\n\n # [DEF:fetch_paginated_count:Function]\n # @PURPOSE: Получает общее количество элементов для пагинации.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: query_params (Dict) - Параметры запроса.\n # @PARAM: count_field (str) - Поле с количеством.\n # @PRE: query_params must be a dictionary.\n # @POST: Returns total count of items.\n # @RETURN: int - Количество.\n def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = \"count\") -> int:\n with belief_scope(\"fetch_paginated_count\"):\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query_params)}))\n return response_json.get(count_field, 0)\n # [/DEF:fetch_paginated_count:Function]\n\n # [DEF:fetch_paginated_data:Function]\n # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.\n # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.\n # @POST: Returns all items across all pages.\n # @RETURN: List[Any] - Список данных.\n def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:\n with belief_scope(\"fetch_paginated_data\"):\n base_query = pagination_options[\"base_query\"]\n total_count = pagination_options.get(\"total_count\")\n \n results_field = pagination_options[\"results_field\"]\n count_field = pagination_options.get(\"count_field\", \"count\")\n page_size = base_query.get('page_size', 1000)\n assert page_size > 0, \"'page_size' must be a positive number.\"\n \n results = []\n page = 0\n \n # Fetch first page to get data and total count if not provided\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n \n first_page_results = response_json.get(results_field, [])\n results.extend(first_page_results)\n \n if total_count is None:\n total_count = response_json.get(count_field, len(first_page_results))\n app_logger.debug(f\"[fetch_paginated_data][State] Total count resolved from first page: {total_count}\")\n\n # Fetch remaining pages\n total_pages = (total_count + page_size - 1) // page_size\n for page in range(1, total_pages):\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n results.extend(response_json.get(results_field, []))\n \n return results\n # [/DEF:fetch_paginated_data:Function]\n\n# [/DEF:APIClient:Class]\n" + "body": "# [DEF:APIClient:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Synchronous Superset API client with process-local auth token caching.\n# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]\n# @RELATION: [DEPENDS_ON] ->[LoggerModule]\nclass APIClient:\n DEFAULT_TIMEOUT = 30\n\n # [DEF:APIClient.__init__:Function]\n # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.\n # @PARAM: config (Dict[str, Any]) - Конфигурация.\n # @PARAM: verify_ssl (bool) - Проверять ли SSL.\n # @PARAM: timeout (int) - Таймаут запросов.\n # @PRE: config must contain 'base_url' and 'auth'.\n # @POST: APIClient instance is initialized with a session.\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n with belief_scope(\"__init__\"):\n log.reason(\"Initializing APIClient\")\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self.session = self._init_session()\n self._tokens: Dict[str, str] = {}\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n self._authenticated = False\n log.reflect(\"APIClient initialized\")\n # [/DEF:APIClient.__init__:Function]\n\n # [DEF:_init_session:Function]\n # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.\n # @PRE: self.request_settings must be initialized.\n # @POST: Returns a configured requests.Session instance.\n # @RETURN: requests.Session - Настроенная сессия.\n def _init_session(self) -> requests.Session:\n with belief_scope(\"_init_session\"):\n session = requests.Session()\n \n # Create a custom adapter that handles TLS issues\n class TLSAdapter(HTTPAdapter):\n def init_poolmanager(self, connections, maxsize, block=False):\n from urllib3.poolmanager import PoolManager\n import ssl\n \n # Create an SSL context that ignores TLSv1 unrecognized name errors\n ctx = ssl.create_default_context()\n ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')\n \n # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification\n # This is safe when verify_ssl is false (we're already not verifying the certificate)\n ctx.check_hostname = False\n \n self.poolmanager = PoolManager(\n num_pools=connections,\n maxsize=maxsize,\n block=block,\n ssl_context=ctx\n )\n \n retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])\n adapter = TLSAdapter(max_retries=retries)\n session.mount('http://', adapter)\n session.mount('https://', adapter)\n \n if not self.request_settings[\"verify_ssl\"]:\n urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n log.reflect(\"SSL verification disabled for session\", payload={\"base_url\": self.base_url})\n # When verify_ssl is false, we should also disable hostname verification\n session.verify = False\n else:\n session.verify = True\n \n return session\n # [/DEF:_init_session:Function]\n\n # [DEF:_normalize_base_url:Function]\n # @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.\n # @PRE: raw_url can be empty.\n # @POST: Returns canonical base URL suitable for building API endpoints.\n # @RETURN: str\n def _normalize_base_url(self, raw_url: str) -> str:\n normalized = str(raw_url or \"\").strip().rstrip(\"/\")\n if normalized.lower().endswith(\"/api/v1\"):\n normalized = normalized[:-len(\"/api/v1\")]\n return normalized.rstrip(\"/\")\n # [/DEF:_normalize_base_url:Function]\n\n # [DEF:_build_api_url:Function]\n # @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.\n # @PRE: endpoint is relative path or absolute URL.\n # @POST: Returns full URL without accidental duplicate slashes.\n # @RETURN: str\n def _build_api_url(self, endpoint: str) -> str:\n normalized_endpoint = str(endpoint or \"\").strip()\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n return normalized_endpoint\n if not normalized_endpoint.startswith(\"/\"):\n normalized_endpoint = f\"/{normalized_endpoint}\"\n if normalized_endpoint.startswith(\"/api/v1/\") or normalized_endpoint == \"/api/v1\":\n return f\"{self.base_url}{normalized_endpoint}\"\n return f\"{self.api_base_url}{normalized_endpoint}\"\n # [/DEF:_build_api_url:Function]\n\n # [DEF:APIClient.authenticate:Function]\n # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.\n # @PRE: self.auth and self.base_url must be valid.\n # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.\n # @RETURN: Dict[str, str] - Словарь с токенами.\n # @THROW: AuthenticationError, NetworkError - при ошибках.\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.\n # When a new APIClient instance reuses cached tokens, its fresh requests.Session\n # has no session cookie — causing superset CSRF check to fail with\n # \"CSRF session token is missing\". The fix is to always fetch a fresh CSRF token\n # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie\n # in the new session while avoiding a full POST login.\n # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing\n # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"authenticate\"):\n log.reason(\"Authenticating to Superset\", payload={\"base_url\": self.base_url})\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this session.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n timeout=self.request_settings[\"timeout\"],\n )\n csrf_response.raise_for_status()\n # Update CSRF token (may have rotated)\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reason(\"Reused cached tokens + refreshed CSRF token\", payload={\"base_url\": self.base_url})\n except Exception as csrf_err:\n # CSRF refresh failed — likely access_token expired, do full re-auth\n log.explore(\"CSRF refresh failed, performing full re-auth\", error=str(csrf_err))\n SupersetAuthCache.invalidate(self._auth_cache_key)\n # Fall through to full login below\n self._tokens = {}\n self._authenticated = False\n\n if not self._authenticated:\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n # Log the payload keys and values (masking password)\n masked_auth = {k: (\"******\" if k == \"password\" else v) for k, v in self.auth.items()}\n log.reason(\"Performing full Superset login\", payload={\"login_url\": login_url, \"auth_payload\": masked_auth})\n\n response = self.session.post(login_url, json=self.auth, timeout=self.request_settings[\"timeout\"])\n\n if response.status_code != 200:\n log.explore(\"Superset login returned non-200 status\", payload={\"status_code\": response.status_code, \"response\": response.text}, error=f\"HTTP {response.status_code}: {response.text[:200]}\")\n\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(csrf_url, headers={\"Authorization\": f\"Bearer {access_token}\"}, timeout=self.request_settings[\"timeout\"])\n csrf_response.raise_for_status()\n\n self._tokens = {\"access_token\": access_token, \"csrf_token\": csrf_response.json()[\"result\"]}\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reflect(\"Authenticated successfully\")\n return self._tokens\n except requests.exceptions.HTTPError as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = e.response.status_code if e.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable during authentication (Status {status_code})\", status_code=status_code) from e\n raise AuthenticationError(f\"Authentication failed: {e}\") from e\n except (requests.exceptions.RequestException, KeyError) as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {e}\") from e\n\n self._authenticated = True\n log.reflect(\"Authenticated successfully (from cached tokens)\")\n return self._tokens\n # [/DEF:APIClient.authenticate:Function]\n\n @property\n # [DEF:headers:Function]\n # @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.\n # @PRE: APIClient is initialized and authenticated or can be authenticated.\n # @POST: Returns headers including auth tokens.\n def headers(self) -> Dict[str, str]:\n if not self._authenticated:\n self.authenticate()\n return {\n \"Authorization\": f\"Bearer {self._tokens['access_token']}\",\n \"X-CSRFToken\": self._tokens.get(\"csrf_token\", \"\"),\n \"Referer\": self.base_url,\n \"Content-Type\": \"application/json\"\n }\n # [/DEF:headers:Function]\n\n # [DEF:request:Function]\n # @PURPOSE: Выполняет универсальный HTTP-запрос к API.\n # @PARAM: method (str) - HTTP метод.\n # @PARAM: endpoint (str) - API эндпоинт.\n # @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.\n # @PARAM: raw_response (bool) - Возвращать ли сырой ответ.\n # @PRE: method and endpoint must be strings.\n # @POST: Returns response content or raw Response object.\n # @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`.\n # @THROW: SupersetAPIError, NetworkError и их подклассы.\n def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]:\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n if headers:\n _headers.update(headers)\n \n try:\n response = self.session.request(method, full_url, headers=_headers, **kwargs)\n response.raise_for_status()\n return response if raw_response else response.json()\n except requests.exceptions.HTTPError as e:\n if e.response is not None and e.response.status_code == 401:\n self._authenticated = False\n self._tokens = {}\n SupersetAuthCache.invalidate(self._auth_cache_key)\n self._handle_http_error(e, endpoint)\n except requests.exceptions.RequestException as e:\n self._handle_network_error(e, full_url)\n # [/DEF:request:Function]\n\n # [DEF:_handle_http_error:Function]\n # @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.\n # @PARAM: e (requests.exceptions.HTTPError) - Ошибка.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PRE: e must be a valid HTTPError with a response.\n # @POST: Raises a specific SupersetAPIError or subclass.\n def _handle_http_error(self, e: requests.exceptions.HTTPError, endpoint: str):\n with belief_scope(\"_handle_http_error\"):\n status_code = e.response.status_code\n if status_code == 502 or status_code == 503 or status_code == 504:\n raise NetworkError(f\"Environment unavailable (Status {status_code})\", status_code=status_code) from e\n if status_code == 404:\n if self._is_dashboard_endpoint(endpoint):\n raise DashboardNotFoundError(endpoint) from e\n raise SupersetAPIError(\n f\"API resource not found at endpoint '{endpoint}'\",\n status_code=status_code,\n endpoint=endpoint,\n subtype=\"not_found\",\n ) from e\n if status_code == 403:\n raise PermissionDeniedError() from e\n if status_code == 401:\n raise AuthenticationError() from e\n raise SupersetAPIError(f\"API Error {status_code}: {e.response.text}\") from e\n # [/DEF:_handle_http_error:Function]\n\n # [DEF:_is_dashboard_endpoint:Function]\n # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.\n # @PRE: endpoint may be relative or absolute.\n # @POST: Returns true only for dashboard-specific endpoints.\n def _is_dashboard_endpoint(self, endpoint: str) -> bool:\n normalized_endpoint = str(endpoint or \"\").strip().lower()\n if not normalized_endpoint:\n return False\n if normalized_endpoint.startswith(\"http://\") or normalized_endpoint.startswith(\"https://\"):\n try:\n normalized_endpoint = \"/\" + normalized_endpoint.split(\"/api/v1\", 1)[1].lstrip(\"/\")\n except IndexError:\n return False\n if normalized_endpoint.startswith(\"/api/v1/\"):\n normalized_endpoint = normalized_endpoint[len(\"/api/v1\"):]\n return normalized_endpoint.startswith(\"/dashboard/\") or normalized_endpoint == \"/dashboard\"\n # [/DEF:_is_dashboard_endpoint:Function]\n\n # [DEF:_handle_network_error:Function]\n # @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.\n # @PARAM: e (requests.exceptions.RequestException) - Ошибка.\n # @PARAM: url (str) - URL.\n # @PRE: e must be a RequestException.\n # @POST: Raises a NetworkError.\n def _handle_network_error(self, e: requests.exceptions.RequestException, url: str):\n with belief_scope(\"_handle_network_error\"):\n if isinstance(e, requests.exceptions.Timeout):\n msg = \"Request timeout\"\n elif isinstance(e, requests.exceptions.ConnectionError):\n msg = \"Connection error\"\n else:\n msg = f\"Unknown network error: {e}\"\n raise NetworkError(msg, url=url) from e\n # [/DEF:_handle_network_error:Function]\n\n # [DEF:upload_file:Function]\n # @PURPOSE: Загружает файл на сервер через multipart/form-data.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: file_info (Dict[str, Any]) - Информация о файле.\n # @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: file_info must contain 'file_obj' and 'file_name'.\n # @POST: File is uploaded and response returned.\n # @RETURN: Ответ API в виде словаря.\n # @THROW: SupersetAPIError, NetworkError, TypeError.\n def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict:\n with belief_scope(\"upload_file\"):\n full_url = self._build_api_url(endpoint)\n _headers = self.headers.copy()\n _headers.pop('Content-Type', None)\n \n \n file_obj, file_name, form_field = file_info.get(\"file_obj\"), file_info.get(\"file_name\"), file_info.get(\"form_field\", \"file\")\n \n files_payload = {}\n if isinstance(file_obj, (str, Path)):\n with open(file_obj, 'rb') as f:\n files_payload = {form_field: (file_name, f.read(), 'application/x-zip-compressed')}\n elif isinstance(file_obj, io.BytesIO):\n files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')}\n else:\n raise TypeError(f\"Unsupported file_obj type: {type(file_obj)}\")\n \n return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)\n # [/DEF:upload_file:Function]\n\n # [DEF:_perform_upload:Function]\n # @PURPOSE: (Helper) Выполняет POST запрос с файлом.\n # @PARAM: url (str) - URL.\n # @PARAM: files (Dict) - Файлы.\n # @PARAM: data (Optional[Dict]) - Данные.\n # @PARAM: headers (Dict) - Заголовки.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: url, files, and headers must be provided.\n # @POST: POST request is performed and JSON response returned.\n # @RETURN: Dict - Ответ.\n def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:\n with belief_scope(\"_perform_upload\"):\n try:\n response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings[\"timeout\"])\n response.raise_for_status()\n if response.status_code == 200:\n try:\n return response.json()\n except Exception as json_e:\n log.explore(\"Upload response not valid JSON\", payload={\"preview\": response.text[:200]}, error=str(json_e))\n raise SupersetAPIError(f\"API error during upload: Response is not valid JSON: {json_e}\") from json_e\n return response.json()\n except requests.exceptions.HTTPError as e:\n raise SupersetAPIError(f\"API error during upload: {e.response.text}\") from e\n except requests.exceptions.RequestException as e:\n raise NetworkError(f\"Network error during upload: {e}\", url=url) from e\n # [/DEF:_perform_upload:Function]\n\n # [DEF:fetch_paginated_count:Function]\n # @PURPOSE: Получает общее количество элементов для пагинации.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: query_params (Dict) - Параметры запроса.\n # @PARAM: count_field (str) - Поле с количеством.\n # @PRE: query_params must be a dictionary.\n # @POST: Returns total count of items.\n # @RETURN: int - Количество.\n def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = \"count\") -> int:\n with belief_scope(\"fetch_paginated_count\"):\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query_params)}))\n return response_json.get(count_field, 0)\n # [/DEF:fetch_paginated_count:Function]\n\n # [DEF:fetch_paginated_data:Function]\n # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.\n # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.\n # @POST: Returns all items across all pages.\n # @RETURN: List[Any] - Список данных.\n def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:\n with belief_scope(\"fetch_paginated_data\"):\n base_query = pagination_options[\"base_query\"]\n total_count = pagination_options.get(\"total_count\")\n \n results_field = pagination_options[\"results_field\"]\n count_field = pagination_options.get(\"count_field\", \"count\")\n page_size = base_query.get('page_size', 1000)\n assert page_size > 0, \"'page_size' must be a positive number.\"\n \n results = []\n page = 0\n \n # Fetch first page to get data and total count if not provided\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n \n first_page_results = response_json.get(results_field, [])\n results.extend(first_page_results)\n \n if total_count is None:\n total_count = response_json.get(count_field, len(first_page_results))\n log.reason(\"Total count resolved from first page\", payload={\"total_count\": total_count})\n\n # Fetch remaining pages\n total_pages = (total_count + page_size - 1) // page_size\n for page in range(1, total_pages):\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n results.extend(response_json.get(results_field, []))\n \n return results\n # [/DEF:fetch_paginated_data:Function]\n\n# [/DEF:APIClient:Class]\n" }, { "contract_id": "APIClient.__init__", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 163, - "end_line": 186, + "start_line": 166, + "end_line": 189, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50234,14 +50990,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:APIClient.__init__:Function]\n # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.\n # @PARAM: config (Dict[str, Any]) - Конфигурация.\n # @PARAM: verify_ssl (bool) - Проверять ли SSL.\n # @PARAM: timeout (int) - Таймаут запросов.\n # @PRE: config must contain 'base_url' and 'auth'.\n # @POST: APIClient instance is initialized with a session.\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n with belief_scope(\"__init__\"):\n app_logger.info(\"[APIClient.__init__][Entry] Initializing APIClient.\")\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self.session = self._init_session()\n self._tokens: Dict[str, str] = {}\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n self._authenticated = False\n app_logger.info(\"[APIClient.__init__][Exit] APIClient initialized.\")\n # [/DEF:APIClient.__init__:Function]\n" + "body": " # [DEF:APIClient.__init__:Function]\n # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.\n # @PARAM: config (Dict[str, Any]) - Конфигурация.\n # @PARAM: verify_ssl (bool) - Проверять ли SSL.\n # @PARAM: timeout (int) - Таймаут запросов.\n # @PRE: config must contain 'base_url' and 'auth'.\n # @POST: APIClient instance is initialized with a session.\n def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):\n with belief_scope(\"__init__\"):\n log.reason(\"Initializing APIClient\")\n self.base_url: str = self._normalize_base_url(config.get(\"base_url\", \"\"))\n self.api_base_url: str = f\"{self.base_url}/api/v1\"\n self.auth = config.get(\"auth\")\n self.request_settings = {\"verify_ssl\": verify_ssl, \"timeout\": timeout}\n self.session = self._init_session()\n self._tokens: Dict[str, str] = {}\n self._auth_cache_key = SupersetAuthCache.build_key(\n self.base_url,\n self.auth,\n verify_ssl,\n )\n self._authenticated = False\n log.reflect(\"APIClient initialized\")\n # [/DEF:APIClient.__init__:Function]\n" }, { "contract_id": "_init_session", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 188, - "end_line": 232, + "start_line": 191, + "end_line": 235, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50278,14 +51034,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_init_session:Function]\n # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.\n # @PRE: self.request_settings must be initialized.\n # @POST: Returns a configured requests.Session instance.\n # @RETURN: requests.Session - Настроенная сессия.\n def _init_session(self) -> requests.Session:\n with belief_scope(\"_init_session\"):\n session = requests.Session()\n \n # Create a custom adapter that handles TLS issues\n class TLSAdapter(HTTPAdapter):\n def init_poolmanager(self, connections, maxsize, block=False):\n from urllib3.poolmanager import PoolManager\n import ssl\n \n # Create an SSL context that ignores TLSv1 unrecognized name errors\n ctx = ssl.create_default_context()\n ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')\n \n # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification\n # This is safe when verify_ssl is false (we're already not verifying the certificate)\n ctx.check_hostname = False\n \n self.poolmanager = PoolManager(\n num_pools=connections,\n maxsize=maxsize,\n block=block,\n ssl_context=ctx\n )\n \n retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])\n adapter = TLSAdapter(max_retries=retries)\n session.mount('http://', adapter)\n session.mount('https://', adapter)\n \n if not self.request_settings[\"verify_ssl\"]:\n urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n app_logger.warning(\"[_init_session][State] SSL verification disabled.\")\n # When verify_ssl is false, we should also disable hostname verification\n session.verify = False\n else:\n session.verify = True\n \n return session\n # [/DEF:_init_session:Function]\n" + "body": " # [DEF:_init_session:Function]\n # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.\n # @PRE: self.request_settings must be initialized.\n # @POST: Returns a configured requests.Session instance.\n # @RETURN: requests.Session - Настроенная сессия.\n def _init_session(self) -> requests.Session:\n with belief_scope(\"_init_session\"):\n session = requests.Session()\n \n # Create a custom adapter that handles TLS issues\n class TLSAdapter(HTTPAdapter):\n def init_poolmanager(self, connections, maxsize, block=False):\n from urllib3.poolmanager import PoolManager\n import ssl\n \n # Create an SSL context that ignores TLSv1 unrecognized name errors\n ctx = ssl.create_default_context()\n ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA')\n \n # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification\n # This is safe when verify_ssl is false (we're already not verifying the certificate)\n ctx.check_hostname = False\n \n self.poolmanager = PoolManager(\n num_pools=connections,\n maxsize=maxsize,\n block=block,\n ssl_context=ctx\n )\n \n retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504])\n adapter = TLSAdapter(max_retries=retries)\n session.mount('http://', adapter)\n session.mount('https://', adapter)\n \n if not self.request_settings[\"verify_ssl\"]:\n urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n log.reflect(\"SSL verification disabled for session\", payload={\"base_url\": self.base_url})\n # When verify_ssl is false, we should also disable hostname verification\n session.verify = False\n else:\n session.verify = True\n \n return session\n # [/DEF:_init_session:Function]\n" }, { "contract_id": "_normalize_base_url", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 234, - "end_line": 244, + "start_line": 237, + "end_line": 247, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50328,8 +51084,8 @@ "contract_id": "_build_api_url", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 246, - "end_line": 260, + "start_line": 249, + "end_line": 263, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50372,8 +51128,8 @@ "contract_id": "APIClient.authenticate", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 262, - "end_line": 345, + "start_line": 265, + "end_line": 347, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50474,14 +51230,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:APIClient.authenticate:Function]\n # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.\n # @PRE: self.auth and self.base_url must be valid.\n # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.\n # @RETURN: Dict[str, str] - Словарь с токенами.\n # @THROW: AuthenticationError, NetworkError - при ошибках.\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.\n # When a new APIClient instance reuses cached tokens, its fresh requests.Session\n # has no session cookie — causing superset CSRF check to fail with\n # \"CSRF session token is missing\". The fix is to always fetch a fresh CSRF token\n # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie\n # in the new session while avoiding a full POST login.\n # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing\n # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"authenticate\"):\n app_logger.info(\"[authenticate][Enter] Authenticating to %s\", self.base_url)\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this session.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n timeout=self.request_settings[\"timeout\"],\n )\n csrf_response.raise_for_status()\n # Update CSRF token (may have rotated)\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s\", self.base_url)\n except Exception as csrf_err:\n # CSRF refresh failed — likely access_token expired, do full re-auth\n app_logger.warning(\"[authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s\", csrf_err)\n SupersetAuthCache.invalidate(self._auth_cache_key)\n # Fall through to full login below\n self._tokens = {}\n self._authenticated = False\n\n if not self._authenticated:\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n # Log the payload keys and values (masking password)\n masked_auth = {k: (\"******\" if k == \"password\" else v) for k, v in self.auth.items()}\n app_logger.info(f\"[authenticate][Debug] Login URL: {login_url}\")\n app_logger.info(f\"[authenticate][Debug] Auth payload: {masked_auth}\")\n\n response = self.session.post(login_url, json=self.auth, timeout=self.request_settings[\"timeout\"])\n\n if response.status_code != 200:\n app_logger.error(f\"[authenticate][Error] Status: {response.status_code}, Response: {response.text}\")\n\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(csrf_url, headers={\"Authorization\": f\"Bearer {access_token}\"}, timeout=self.request_settings[\"timeout\"])\n csrf_response.raise_for_status()\n\n self._tokens = {\"access_token\": access_token, \"csrf_token\": csrf_response.json()[\"result\"]}\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n app_logger.info(\"[authenticate][Exit] Authenticated successfully.\")\n return self._tokens\n except requests.exceptions.HTTPError as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = e.response.status_code if e.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable during authentication (Status {status_code})\", status_code=status_code) from e\n raise AuthenticationError(f\"Authentication failed: {e}\") from e\n except (requests.exceptions.RequestException, KeyError) as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {e}\") from e\n\n self._authenticated = True\n app_logger.info(\"[authenticate][Exit] Authenticated successfully (from cache).\")\n return self._tokens\n # [/DEF:APIClient.authenticate:Function]\n" + "body": " # [DEF:APIClient.authenticate:Function]\n # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.\n # @PRE: self.auth and self.base_url must be valid.\n # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.\n # @RETURN: Dict[str, str] - Словарь с токенами.\n # @THROW: AuthenticationError, NetworkError - при ошибках.\n # @RELATION: [CALLS] ->[SupersetAuthCache.get]\n # @RELATION: [CALLS] ->[SupersetAuthCache.set]\n # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.\n # When a new APIClient instance reuses cached tokens, its fresh requests.Session\n # has no session cookie — causing superset CSRF check to fail with\n # \"CSRF session token is missing\". The fix is to always fetch a fresh CSRF token\n # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie\n # in the new session while avoiding a full POST login.\n # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing\n # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.\n def authenticate(self) -> Dict[str, str]:\n with belief_scope(\"authenticate\"):\n log.reason(\"Authenticating to Superset\", payload={\"base_url\": self.base_url})\n cached_tokens = SupersetAuthCache.get(self._auth_cache_key)\n\n if cached_tokens and cached_tokens.get(\"access_token\") and cached_tokens.get(\"csrf_token\"):\n self._tokens = cached_tokens\n # Always fetch a fresh CSRF token to establish session cookie in this session.\n # The cached access_token may still be valid; we avoid a full POST login.\n try:\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(\n csrf_url,\n headers={\"Authorization\": f\"Bearer {self._tokens['access_token']}\"},\n timeout=self.request_settings[\"timeout\"],\n )\n csrf_response.raise_for_status()\n # Update CSRF token (may have rotated)\n self._tokens[\"csrf_token\"] = csrf_response.json()[\"result\"]\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reason(\"Reused cached tokens + refreshed CSRF token\", payload={\"base_url\": self.base_url})\n except Exception as csrf_err:\n # CSRF refresh failed — likely access_token expired, do full re-auth\n log.explore(\"CSRF refresh failed, performing full re-auth\", error=str(csrf_err))\n SupersetAuthCache.invalidate(self._auth_cache_key)\n # Fall through to full login below\n self._tokens = {}\n self._authenticated = False\n\n if not self._authenticated:\n try:\n login_url = f\"{self.api_base_url}/security/login\"\n # Log the payload keys and values (masking password)\n masked_auth = {k: (\"******\" if k == \"password\" else v) for k, v in self.auth.items()}\n log.reason(\"Performing full Superset login\", payload={\"login_url\": login_url, \"auth_payload\": masked_auth})\n\n response = self.session.post(login_url, json=self.auth, timeout=self.request_settings[\"timeout\"])\n\n if response.status_code != 200:\n log.explore(\"Superset login returned non-200 status\", payload={\"status_code\": response.status_code, \"response\": response.text}, error=f\"HTTP {response.status_code}: {response.text[:200]}\")\n\n response.raise_for_status()\n access_token = response.json()[\"access_token\"]\n\n csrf_url = f\"{self.api_base_url}/security/csrf_token/\"\n csrf_response = self.session.get(csrf_url, headers={\"Authorization\": f\"Bearer {access_token}\"}, timeout=self.request_settings[\"timeout\"])\n csrf_response.raise_for_status()\n\n self._tokens = {\"access_token\": access_token, \"csrf_token\": csrf_response.json()[\"result\"]}\n self._authenticated = True\n SupersetAuthCache.set(self._auth_cache_key, self._tokens)\n log.reflect(\"Authenticated successfully\")\n return self._tokens\n except requests.exceptions.HTTPError as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n status_code = e.response.status_code if e.response is not None else None\n if status_code in [502, 503, 504]:\n raise NetworkError(f\"Environment unavailable during authentication (Status {status_code})\", status_code=status_code) from e\n raise AuthenticationError(f\"Authentication failed: {e}\") from e\n except (requests.exceptions.RequestException, KeyError) as e:\n SupersetAuthCache.invalidate(self._auth_cache_key)\n raise NetworkError(f\"Network or parsing error during authentication: {e}\") from e\n\n self._authenticated = True\n log.reflect(\"Authenticated successfully (from cached tokens)\")\n return self._tokens\n # [/DEF:APIClient.authenticate:Function]\n" }, { "contract_id": "headers", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 348, - "end_line": 361, + "start_line": 350, + "end_line": 363, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50517,8 +51273,8 @@ "contract_id": "request", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 363, - "end_line": 391, + "start_line": 365, + "end_line": 393, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50575,8 +51331,8 @@ "contract_id": "_handle_http_error", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 393, - "end_line": 418, + "start_line": 395, + "end_line": 420, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50619,8 +51375,8 @@ "contract_id": "_is_dashboard_endpoint", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 420, - "end_line": 436, + "start_line": 422, + "end_line": 438, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50656,8 +51412,8 @@ "contract_id": "_handle_network_error", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 438, - "end_line": 453, + "start_line": 440, + "end_line": 455, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50700,8 +51456,8 @@ "contract_id": "upload_file", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 455, - "end_line": 484, + "start_line": 457, + "end_line": 486, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50758,8 +51514,8 @@ "contract_id": "_perform_upload", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 486, - "end_line": 512, + "start_line": 488, + "end_line": 514, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50803,14 +51559,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_perform_upload:Function]\n # @PURPOSE: (Helper) Выполняет POST запрос с файлом.\n # @PARAM: url (str) - URL.\n # @PARAM: files (Dict) - Файлы.\n # @PARAM: data (Optional[Dict]) - Данные.\n # @PARAM: headers (Dict) - Заголовки.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: url, files, and headers must be provided.\n # @POST: POST request is performed and JSON response returned.\n # @RETURN: Dict - Ответ.\n def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:\n with belief_scope(\"_perform_upload\"):\n try:\n response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings[\"timeout\"])\n response.raise_for_status()\n if response.status_code == 200:\n try:\n return response.json()\n except Exception as json_e:\n app_logger.debug(f\"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...\")\n raise SupersetAPIError(f\"API error during upload: Response is not valid JSON: {json_e}\") from json_e\n return response.json()\n except requests.exceptions.HTTPError as e:\n raise SupersetAPIError(f\"API error during upload: {e.response.text}\") from e\n except requests.exceptions.RequestException as e:\n raise NetworkError(f\"Network error during upload: {e}\", url=url) from e\n # [/DEF:_perform_upload:Function]\n" + "body": " # [DEF:_perform_upload:Function]\n # @PURPOSE: (Helper) Выполняет POST запрос с файлом.\n # @PARAM: url (str) - URL.\n # @PARAM: files (Dict) - Файлы.\n # @PARAM: data (Optional[Dict]) - Данные.\n # @PARAM: headers (Dict) - Заголовки.\n # @PARAM: timeout (Optional[int]) - Таймаут.\n # @PRE: url, files, and headers must be provided.\n # @POST: POST request is performed and JSON response returned.\n # @RETURN: Dict - Ответ.\n def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict:\n with belief_scope(\"_perform_upload\"):\n try:\n response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings[\"timeout\"])\n response.raise_for_status()\n if response.status_code == 200:\n try:\n return response.json()\n except Exception as json_e:\n log.explore(\"Upload response not valid JSON\", payload={\"preview\": response.text[:200]}, error=str(json_e))\n raise SupersetAPIError(f\"API error during upload: Response is not valid JSON: {json_e}\") from json_e\n return response.json()\n except requests.exceptions.HTTPError as e:\n raise SupersetAPIError(f\"API error during upload: {e.response.text}\") from e\n except requests.exceptions.RequestException as e:\n raise NetworkError(f\"Network error during upload: {e}\", url=url) from e\n # [/DEF:_perform_upload:Function]\n" }, { "contract_id": "fetch_paginated_count", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 514, - "end_line": 526, + "start_line": 516, + "end_line": 528, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50860,8 +51616,8 @@ "contract_id": "fetch_paginated_data", "contract_type": "Function", "file_path": "backend/src/core/utils/network.py", - "start_line": 528, - "end_line": 567, + "start_line": 530, + "end_line": 569, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -50905,7 +51661,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:fetch_paginated_data:Function]\n # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.\n # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.\n # @POST: Returns all items across all pages.\n # @RETURN: List[Any] - Список данных.\n def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:\n with belief_scope(\"fetch_paginated_data\"):\n base_query = pagination_options[\"base_query\"]\n total_count = pagination_options.get(\"total_count\")\n \n results_field = pagination_options[\"results_field\"]\n count_field = pagination_options.get(\"count_field\", \"count\")\n page_size = base_query.get('page_size', 1000)\n assert page_size > 0, \"'page_size' must be a positive number.\"\n \n results = []\n page = 0\n \n # Fetch first page to get data and total count if not provided\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n \n first_page_results = response_json.get(results_field, [])\n results.extend(first_page_results)\n \n if total_count is None:\n total_count = response_json.get(count_field, len(first_page_results))\n app_logger.debug(f\"[fetch_paginated_data][State] Total count resolved from first page: {total_count}\")\n\n # Fetch remaining pages\n total_pages = (total_count + page_size - 1) // page_size\n for page in range(1, total_pages):\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n results.extend(response_json.get(results_field, []))\n \n return results\n # [/DEF:fetch_paginated_data:Function]\n" + "body": " # [DEF:fetch_paginated_data:Function]\n # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.\n # @PARAM: endpoint (str) - Эндпоинт.\n # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.\n # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.\n # @POST: Returns all items across all pages.\n # @RETURN: List[Any] - Список данных.\n def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:\n with belief_scope(\"fetch_paginated_data\"):\n base_query = pagination_options[\"base_query\"]\n total_count = pagination_options.get(\"total_count\")\n \n results_field = pagination_options[\"results_field\"]\n count_field = pagination_options.get(\"count_field\", \"count\")\n page_size = base_query.get('page_size', 1000)\n assert page_size > 0, \"'page_size' must be a positive number.\"\n \n results = []\n page = 0\n \n # Fetch first page to get data and total count if not provided\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n \n first_page_results = response_json.get(results_field, [])\n results.extend(first_page_results)\n \n if total_count is None:\n total_count = response_json.get(count_field, len(first_page_results))\n log.reason(\"Total count resolved from first page\", payload={\"total_count\": total_count})\n\n # Fetch remaining pages\n total_pages = (total_count + page_size - 1) // page_size\n for page in range(1, total_pages):\n query = {**base_query, 'page': page}\n response_json = cast(Dict[str, Any], self.request(\"GET\", endpoint, params={\"q\": json.dumps(query)}))\n results.extend(response_json.get(results_field, []))\n \n return results\n # [/DEF:fetch_paginated_data:Function]\n" }, { "contract_id": "SupersetCompilationAdapter", @@ -52434,12 +53190,265 @@ "anchor_syntax": "def", "body": " # [DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.\n def _normalize_default_literal(self, literal: Optional[str]) -> Any:\n normalized_literal = str(literal or \"\").strip()\n if not normalized_literal:\n return None\n if (\n normalized_literal.startswith(\"'\") and normalized_literal.endswith(\"'\")\n ) or (\n normalized_literal.startswith('\"') and normalized_literal.endswith('\"')\n ):\n return normalized_literal[1:-1]\n lowered = normalized_literal.lower()\n if lowered in {\"true\", \"false\"}:\n return lowered == \"true\"\n if lowered in {\"null\", \"none\"}:\n return None\n try:\n return int(normalized_literal)\n except ValueError:\n try:\n return float(normalized_literal)\n except ValueError:\n return normalized_literal\n\n # [/DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]\n" }, + { + "contract_id": "WsLogHandlerModule", + "contract_type": "Module", + "file_path": "backend/src/core/ws_log_handler.py", + "start_line": 1, + "end_line": 70, + "tier": "TIER_2", + "complexity": 3, + "metadata": { + "BRIEF": "WebSocket log handler module providing LogEntry DTO and WebSocketLogHandler for buffered log streaming.", + "COMPLEXITY": 3 + }, + "relations": [ + { + "source_id": "WsLogHandlerModule", + "relation_type": "DEPENDS_ON", + "target_id": "LogEntry", + "target_ref": "[LogEntry]" + }, + { + "source_id": "WsLogHandlerModule", + "relation_type": "COMPOSES", + "target_id": "CotJsonFormatter", + "target_ref": "[CotJsonFormatter]" + } + ], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "LAYER", + "message": "@LAYER is required for contract type 'Module' at C3", + "detail": { + "actual_complexity": 3, + "contract_type": "Module" + } + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Module' at C3", + "detail": { + "actual_complexity": 3, + "contract_type": "Module" + } + }, + { + "code": "invalid_relation_predicate", + "tag": "RELATION", + "message": "Predicate COMPOSES is not in allowed_predicates", + "detail": { + "allowed": [ + "DEPENDS_ON", + "CALLS", + "INHERITS", + "IMPLEMENTS", + "DISPATCHES", + "BINDS_TO", + "VERIFIES" + ], + "predicate": "COMPOSES" + } + } + ], + "anchor_syntax": "region", + "body": "# #region WsLogHandlerModule [C:3] [TYPE Module] [SEMANTICS logging,handler,websocket,buffer]\n# @BRIEF WebSocket log handler module providing LogEntry DTO and WebSocketLogHandler for buffered log streaming.\n# @RELATION DEPENDS_ON -> [LogEntry]\n# @RELATION COMPOSES -> [CotJsonFormatter]\nimport logging\nfrom typing import Dict, Any, List, Optional\nfrom collections import deque\nfrom datetime import datetime\n\nfrom pydantic import BaseModel, Field\n\n\n# #region LogEntry [C:1] [TYPE Class] [SEMANTICS log,entry,record,pydantic]\nclass LogEntry(BaseModel):\n timestamp: datetime = Field(default_factory=datetime.utcnow)\n level: str\n message: str\n context: Optional[Dict[str, Any]] = None\n# #endregion LogEntry\n\n\n# #region WebSocketLogHandler [C:3] [TYPE Class] [SEMANTICS logging,handler,websocket,buffer]\n# @BRIEF Custom logging handler that captures log records into a fixed-capacity buffer for WebSocket streaming.\n# @RELATION DEPENDS_ON -> [LogEntry]\n# @RELATION COMPOSES -> [CotJsonFormatter]\nclass WebSocketLogHandler(logging.Handler):\n \"\"\"\n A logging handler that stores log records and can be extended to send them\n over WebSockets.\n \"\"\"\n\n # #region WebSocketLogHandler.__init__ [C:1] [TYPE Function] [SEMANTICS init,handler,buffer]\n def __init__(self, capacity: int = 1000):\n super().__init__()\n self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)\n # #endregion WebSocketLogHandler.__init__\n\n # #region WebSocketLogHandler.emit [C:2] [TYPE Function] [SEMANTICS log,capture,format,buffer]\n # @BRIEF Captures a log record, formats it, and stores it in the buffer as a LogEntry.\n def emit(self, record: logging.LogRecord):\n try:\n log_entry = LogEntry(\n level=record.levelname,\n message=self.format(record),\n context={\n \"name\": record.name,\n \"pathname\": record.pathname,\n \"lineno\": record.lineno,\n \"funcName\": record.funcName,\n \"process\": record.process,\n \"thread\": record.thread,\n }\n )\n self.log_buffer.append(log_entry)\n except Exception:\n self.handleError(record)\n # #endregion WebSocketLogHandler.emit\n\n # #region WebSocketLogHandler.get_recent_logs [C:2] [TYPE Function] [SEMANTICS log,retrieve,buffer]\n # @BRIEF Returns a list of recent log entries from the buffer.\n def get_recent_logs(self) -> List[LogEntry]:\n \"\"\"\n Returns a list of recent log entries from the buffer.\n \"\"\"\n return list(self.log_buffer)\n # #endregion WebSocketLogHandler.get_recent_logs\n\n\n# #endregion WebSocketLogHandler\n# #endregion WsLogHandlerModule\n" + }, + { + "contract_id": "LogEntry", + "contract_type": "Class", + "file_path": "backend/src/core/ws_log_handler.py", + "start_line": 13, + "end_line": 19, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1 + }, + "relations": [], + "schema_warnings": [ + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Class' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Class" + } + } + ], + "anchor_syntax": "region", + "body": "# #region LogEntry [C:1] [TYPE Class] [SEMANTICS log,entry,record,pydantic]\nclass LogEntry(BaseModel):\n timestamp: datetime = Field(default_factory=datetime.utcnow)\n level: str\n message: str\n context: Optional[Dict[str, Any]] = None\n# #endregion LogEntry\n" + }, + { + "contract_id": "WebSocketLogHandler", + "contract_type": "Class", + "file_path": "backend/src/core/ws_log_handler.py", + "start_line": 22, + "end_line": 69, + "tier": "TIER_2", + "complexity": 3, + "metadata": { + "BRIEF": "Custom logging handler that captures log records into a fixed-capacity buffer for WebSocket streaming.", + "COMPLEXITY": 3 + }, + "relations": [ + { + "source_id": "WebSocketLogHandler", + "relation_type": "DEPENDS_ON", + "target_id": "LogEntry", + "target_ref": "[LogEntry]" + }, + { + "source_id": "WebSocketLogHandler", + "relation_type": "COMPOSES", + "target_id": "CotJsonFormatter", + "target_ref": "[CotJsonFormatter]" + } + ], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Class' at C3", + "detail": { + "actual_complexity": 3, + "contract_type": "Class" + } + }, + { + "code": "invalid_relation_predicate", + "tag": "RELATION", + "message": "Predicate COMPOSES is not in allowed_predicates", + "detail": { + "allowed": [ + "DEPENDS_ON", + "CALLS", + "INHERITS", + "IMPLEMENTS", + "DISPATCHES", + "BINDS_TO", + "VERIFIES" + ], + "predicate": "COMPOSES" + } + } + ], + "anchor_syntax": "region", + "body": "# #region WebSocketLogHandler [C:3] [TYPE Class] [SEMANTICS logging,handler,websocket,buffer]\n# @BRIEF Custom logging handler that captures log records into a fixed-capacity buffer for WebSocket streaming.\n# @RELATION DEPENDS_ON -> [LogEntry]\n# @RELATION COMPOSES -> [CotJsonFormatter]\nclass WebSocketLogHandler(logging.Handler):\n \"\"\"\n A logging handler that stores log records and can be extended to send them\n over WebSockets.\n \"\"\"\n\n # #region WebSocketLogHandler.__init__ [C:1] [TYPE Function] [SEMANTICS init,handler,buffer]\n def __init__(self, capacity: int = 1000):\n super().__init__()\n self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)\n # #endregion WebSocketLogHandler.__init__\n\n # #region WebSocketLogHandler.emit [C:2] [TYPE Function] [SEMANTICS log,capture,format,buffer]\n # @BRIEF Captures a log record, formats it, and stores it in the buffer as a LogEntry.\n def emit(self, record: logging.LogRecord):\n try:\n log_entry = LogEntry(\n level=record.levelname,\n message=self.format(record),\n context={\n \"name\": record.name,\n \"pathname\": record.pathname,\n \"lineno\": record.lineno,\n \"funcName\": record.funcName,\n \"process\": record.process,\n \"thread\": record.thread,\n }\n )\n self.log_buffer.append(log_entry)\n except Exception:\n self.handleError(record)\n # #endregion WebSocketLogHandler.emit\n\n # #region WebSocketLogHandler.get_recent_logs [C:2] [TYPE Function] [SEMANTICS log,retrieve,buffer]\n # @BRIEF Returns a list of recent log entries from the buffer.\n def get_recent_logs(self) -> List[LogEntry]:\n \"\"\"\n Returns a list of recent log entries from the buffer.\n \"\"\"\n return list(self.log_buffer)\n # #endregion WebSocketLogHandler.get_recent_logs\n\n\n# #endregion WebSocketLogHandler\n" + }, + { + "contract_id": "WebSocketLogHandler.__init__", + "contract_type": "Function", + "file_path": "backend/src/core/ws_log_handler.py", + "start_line": 32, + "end_line": 36, + "tier": "TIER_1", + "complexity": 1, + "metadata": { + "COMPLEXITY": 1 + }, + "relations": [], + "schema_warnings": [ + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C1", + "detail": { + "actual_complexity": 1, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": " # #region WebSocketLogHandler.__init__ [C:1] [TYPE Function] [SEMANTICS init,handler,buffer]\n def __init__(self, capacity: int = 1000):\n super().__init__()\n self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)\n # #endregion WebSocketLogHandler.__init__\n" + }, + { + "contract_id": "WebSocketLogHandler.emit", + "contract_type": "Function", + "file_path": "backend/src/core/ws_log_handler.py", + "start_line": 38, + "end_line": 57, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Captures a log record, formats it, and stores it in the buffer as a LogEntry.", + "COMPLEXITY": 2 + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": " # #region WebSocketLogHandler.emit [C:2] [TYPE Function] [SEMANTICS log,capture,format,buffer]\n # @BRIEF Captures a log record, formats it, and stores it in the buffer as a LogEntry.\n def emit(self, record: logging.LogRecord):\n try:\n log_entry = LogEntry(\n level=record.levelname,\n message=self.format(record),\n context={\n \"name\": record.name,\n \"pathname\": record.pathname,\n \"lineno\": record.lineno,\n \"funcName\": record.funcName,\n \"process\": record.process,\n \"thread\": record.thread,\n }\n )\n self.log_buffer.append(log_entry)\n except Exception:\n self.handleError(record)\n # #endregion WebSocketLogHandler.emit\n" + }, + { + "contract_id": "WebSocketLogHandler.get_recent_logs", + "contract_type": "Function", + "file_path": "backend/src/core/ws_log_handler.py", + "start_line": 59, + "end_line": 66, + "tier": "TIER_1", + "complexity": 2, + "metadata": { + "BRIEF": "Returns a list of recent log entries from the buffer.", + "COMPLEXITY": 2 + }, + "relations": [], + "schema_warnings": [ + { + "code": "unknown_tag", + "tag": "BRIEF", + "message": "@BRIEF is not defined in axiom_config.yaml tags", + "detail": null + }, + { + "code": "missing_required_tag", + "tag": "PURPOSE", + "message": "@PURPOSE is required for contract type 'Function' at C2", + "detail": { + "actual_complexity": 2, + "contract_type": "Function" + } + } + ], + "anchor_syntax": "region", + "body": " # #region WebSocketLogHandler.get_recent_logs [C:2] [TYPE Function] [SEMANTICS log,retrieve,buffer]\n # @BRIEF Returns a list of recent log entries from the buffer.\n def get_recent_logs(self) -> List[LogEntry]:\n \"\"\"\n Returns a list of recent log entries from the buffer.\n \"\"\"\n return list(self.log_buffer)\n # #endregion WebSocketLogHandler.get_recent_logs\n" + }, { "contract_id": "AppDependencies", "contract_type": "Module", "file_path": "backend/src/dependencies.py", "start_line": 1, - "end_line": 304, + "end_line": 306, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -52534,14 +53543,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:AppDependencies:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: dependency, injection, singleton, factory, auth, jwt\n# @PURPOSE: Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports.\n# @LAYER: Core\n# @RELATION: Used by main app and API routers to get access to shared instances.\n# @RELATION: CALLS ->[CleanReleaseRepository]\n# @RELATION: CALLS ->[ConfigManager]\n# @RELATION: CALLS ->[PluginLoader]\n# @RELATION: CALLS ->[SchedulerService]\n# @RELATION: CALLS ->[TaskManager]\n# @RELATION: CALLS ->[get_all_plugin_configs]\n# @RELATION: CALLS ->[get_db]\n# @RELATION: CALLS ->[info]\n# @RELATION: CALLS ->[init_db]\n\nfrom pathlib import Path\nfrom typing import Optional\nfrom fastapi import Depends, HTTPException, status\nfrom fastapi.security import OAuth2PasswordBearer\nfrom jose import JWTError\nfrom .core.plugin_loader import PluginLoader\nfrom .core.task_manager import TaskManager\nfrom .core.config_manager import ConfigManager\nfrom .core.scheduler import SchedulerService\nfrom .services.resource_service import ResourceService\nfrom .services.mapping_service import MappingService\nfrom .services.clean_release.repositories import (\n CandidateRepository,\n ArtifactRepository,\n ManifestRepository,\n PolicyRepository,\n ComplianceRepository,\n ReportRepository,\n ApprovalRepository,\n PublicationRepository,\n AuditRepository,\n CleanReleaseAuditLog,\n)\nfrom .services.clean_release.repository import CleanReleaseRepository\nfrom .services.clean_release.facade import CleanReleaseFacade\nfrom .services.reports.report_service import ReportsService\nfrom .core.database import init_db, get_auth_db, get_db\nfrom .core.logger import logger\nfrom .core.auth.jwt import decode_token\nfrom .core.auth.repository import AuthRepository\nfrom .models.auth import User\n\n# Initialize singletons lazily to avoid import-time DB side effects during test collection.\n# Use absolute path relative to this file to ensure plugins are found regardless of CWD\nproject_root = Path(__file__).parent.parent.parent\nconfig_path = project_root / \"config.json\"\n\nconfig_manager: Optional[ConfigManager] = None\nplugin_loader: Optional[PluginLoader] = None\ntask_manager: Optional[TaskManager] = None\nscheduler_service: Optional[SchedulerService] = None\nresource_service: Optional[ResourceService] = None\n\n\n# [DEF:get_config_manager:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for ConfigManager.\n# @PRE: Global config_manager must be initialized.\n# @POST: Returns shared ConfigManager instance.\n# @RETURN: ConfigManager - The shared config manager instance.\ndef get_config_manager() -> ConfigManager:\n \"\"\"Dependency injector for ConfigManager.\"\"\"\n global config_manager\n if config_manager is None:\n init_db()\n config_manager = ConfigManager(config_path=str(config_path))\n return config_manager\n\n\n# [/DEF:get_config_manager:Function]\n\nplugin_dir = Path(__file__).parent / \"plugins\"\n\n# Clean Release Redesign Singletons\n# Note: These use get_db() which is a generator, so we need a way to provide a session.\n# For singletons in dependencies.py, we might need a different approach or\n# initialize them inside the dependency functions.\n\n\n# [DEF:get_plugin_loader:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for PluginLoader.\n# @PRE: Global plugin_loader must be initialized.\n# @POST: Returns shared PluginLoader instance.\n# @RETURN: PluginLoader - The shared plugin loader instance.\ndef get_plugin_loader() -> PluginLoader:\n \"\"\"Dependency injector for PluginLoader.\"\"\"\n global plugin_loader\n if plugin_loader is None:\n plugin_loader = PluginLoader(plugin_dir=str(plugin_dir))\n logger.info(f\"PluginLoader initialized with directory: {plugin_dir}\")\n logger.info(\n f\"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}\"\n )\n return plugin_loader\n\n\n# [/DEF:get_plugin_loader:Function]\n\n\n# [DEF:get_task_manager:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for TaskManager.\n# @PRE: Global task_manager must be initialized.\n# @POST: Returns shared TaskManager instance.\n# @RETURN: TaskManager - The shared task manager instance.\ndef get_task_manager() -> TaskManager:\n \"\"\"Dependency injector for TaskManager.\"\"\"\n global task_manager\n if task_manager is None:\n task_manager = TaskManager(get_plugin_loader())\n logger.info(\"TaskManager initialized\")\n return task_manager\n\n\n# [/DEF:get_task_manager:Function]\n\n\n# [DEF:get_scheduler_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for SchedulerService.\n# @PRE: Global scheduler_service must be initialized.\n# @POST: Returns shared SchedulerService instance.\n# @RETURN: SchedulerService - The shared scheduler service instance.\ndef get_scheduler_service() -> SchedulerService:\n \"\"\"Dependency injector for SchedulerService.\"\"\"\n global scheduler_service\n if scheduler_service is None:\n scheduler_service = SchedulerService(get_task_manager(), get_config_manager())\n logger.info(\"SchedulerService initialized\")\n return scheduler_service\n\n\n# [/DEF:get_scheduler_service:Function]\n\n\n# [DEF:get_resource_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for ResourceService.\n# @PRE: Global resource_service must be initialized.\n# @POST: Returns shared ResourceService instance.\n# @RETURN: ResourceService - The shared resource service instance.\ndef get_resource_service() -> ResourceService:\n \"\"\"Dependency injector for ResourceService.\"\"\"\n global resource_service\n if resource_service is None:\n resource_service = ResourceService()\n logger.info(\"ResourceService initialized\")\n return resource_service\n\n\n# [/DEF:get_resource_service:Function]\n\n\n# [DEF:get_mapping_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for MappingService.\n# @PRE: Global config_manager must be initialized.\n# @POST: Returns new MappingService instance.\n# @RETURN: MappingService - A new mapping service instance.\ndef get_mapping_service() -> MappingService:\n \"\"\"Dependency injector for MappingService.\"\"\"\n return MappingService(get_config_manager())\n\n\n# [/DEF:get_mapping_service:Function]\n\n\n_clean_release_repository = CleanReleaseRepository()\n\n\n# [DEF:get_clean_release_repository:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Legacy compatibility shim for CleanReleaseRepository.\n# @POST: Returns a shared CleanReleaseRepository instance.\ndef get_clean_release_repository() -> CleanReleaseRepository:\n \"\"\"Legacy compatibility shim for CleanReleaseRepository.\"\"\"\n return _clean_release_repository\n\n\n# [/DEF:get_clean_release_repository:Function]\n\n\n# [DEF:get_clean_release_facade:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for CleanReleaseFacade.\n# @POST: Returns a facade instance with a fresh DB session.\ndef get_clean_release_facade(db=Depends(get_db)) -> CleanReleaseFacade:\n candidate_repo = CandidateRepository(db)\n artifact_repo = ArtifactRepository(db)\n manifest_repo = ManifestRepository(db)\n policy_repo = PolicyRepository(db)\n compliance_repo = ComplianceRepository(db)\n report_repo = ReportRepository(db)\n approval_repo = ApprovalRepository(db)\n publication_repo = PublicationRepository(db)\n audit_repo = AuditRepository(db)\n\n return CleanReleaseFacade(\n candidate_repo=candidate_repo,\n artifact_repo=artifact_repo,\n manifest_repo=manifest_repo,\n policy_repo=policy_repo,\n compliance_repo=compliance_repo,\n report_repo=report_repo,\n approval_repo=approval_repo,\n publication_repo=publication_repo,\n audit_repo=audit_repo,\n config_manager=get_config_manager(),\n )\n\n\n# [/DEF:get_clean_release_facade:Function]\n\n# [DEF:oauth2_scheme:Variable]\n# @RELATION: DEPENDS_ON -> OAuth2PasswordBearer\n# @COMPLEXITY: 1\n# @PURPOSE: OAuth2 password bearer scheme for token extraction.\noauth2_scheme = OAuth2PasswordBearer(tokenUrl=\"/api/auth/login\")\n# [/DEF:oauth2_scheme:Variable]\n\n\n# [DEF:get_current_user:Function]\n# @RELATION: CALLS -> AuthRepository\n# @COMPLEXITY: 3\n# @PURPOSE: Dependency for retrieving currently authenticated user from a JWT.\n# @PRE: JWT token provided in Authorization header.\n# @POST: Returns User object if token is valid.\n# @THROW: HTTPException 401 if token is invalid or user not found.\n# @PARAM: token (str) - Extracted JWT token.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: User - The authenticated user.\ndef get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db)):\n credentials_exception = HTTPException(\n status_code=status.HTTP_401_UNAUTHORIZED,\n detail=\"Could not validate credentials\",\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n try:\n payload = decode_token(token)\n username_value = payload.get(\"sub\")\n if not isinstance(username_value, str) or not username_value:\n raise credentials_exception\n username = username_value\n except JWTError:\n raise credentials_exception\n\n repo = AuthRepository(db)\n user = repo.get_user_by_username(username)\n if user is None:\n raise credentials_exception\n return user\n\n\n# [/DEF:get_current_user:Function]\n\n\n# [DEF:has_permission:Function]\n# @RELATION: CALLS -> AuthRepository\n# @COMPLEXITY: 3\n# @PURPOSE: Dependency for checking if the current user has a specific permission.\n# @PRE: User is authenticated.\n# @POST: Returns True if user has permission.\n# @THROW: HTTPException 403 if permission is denied.\n# @PARAM: resource (str) - The resource identifier.\n# @PARAM: action (str) - The action identifier (READ, EXECUTE, WRITE).\n# @RETURN: User - The authenticated user if permission granted.\ndef has_permission(resource: str, action: str):\n def permission_checker(current_user: User = Depends(get_current_user)):\n # Union of all permissions across all roles\n for role in current_user.roles:\n for perm in role.permissions:\n if perm.resource == resource and perm.action == action:\n return current_user\n\n # Special case for Admin role (full access)\n if any(role.name == \"Admin\" for role in current_user.roles):\n return current_user\n\n from .core.auth.logger import log_security_event\n\n log_security_event(\n \"PERMISSION_DENIED\",\n str(getattr(current_user, \"username\", \"unknown\")),\n {\"resource\": resource, \"action\": action},\n )\n\n raise HTTPException(\n status_code=status.HTTP_403_FORBIDDEN,\n detail=f\"Permission denied for {resource}:{action}\",\n )\n\n return permission_checker\n\n\n# [/DEF:has_permission:Function]\n\n# [/DEF:AppDependencies:Module]\n" + "body": "# [DEF:AppDependencies:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: dependency, injection, singleton, factory, auth, jwt\n# @PURPOSE: Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports.\n# @LAYER: Core\n# @RELATION: Used by main app and API routers to get access to shared instances.\n# @RELATION: CALLS ->[CleanReleaseRepository]\n# @RELATION: CALLS ->[ConfigManager]\n# @RELATION: CALLS ->[PluginLoader]\n# @RELATION: CALLS ->[SchedulerService]\n# @RELATION: CALLS ->[TaskManager]\n# @RELATION: CALLS ->[get_all_plugin_configs]\n# @RELATION: CALLS ->[get_db]\n# @RELATION: CALLS ->[info]\n# @RELATION: CALLS ->[init_db]\n\nfrom pathlib import Path\nfrom typing import Optional\nfrom fastapi import Depends, HTTPException, status\nfrom fastapi.security import OAuth2PasswordBearer\nfrom jose import JWTError\nfrom .core.plugin_loader import PluginLoader\nfrom .core.task_manager import TaskManager\nfrom .core.config_manager import ConfigManager\nfrom .core.scheduler import SchedulerService\nfrom .services.resource_service import ResourceService\nfrom .services.mapping_service import MappingService\nfrom .services.clean_release.repositories import (\n CandidateRepository,\n ArtifactRepository,\n ManifestRepository,\n PolicyRepository,\n ComplianceRepository,\n ReportRepository,\n ApprovalRepository,\n PublicationRepository,\n AuditRepository,\n CleanReleaseAuditLog,\n)\nfrom .services.clean_release.repository import CleanReleaseRepository\nfrom .services.clean_release.facade import CleanReleaseFacade\nfrom .services.reports.report_service import ReportsService\nfrom .core.database import init_db, get_auth_db, get_db\nfrom .core.cot_logger import MarkerLogger\nfrom .core.auth.jwt import decode_token\nfrom .core.auth.repository import AuthRepository\nfrom .models.auth import User\n\nlog = MarkerLogger(\"Dependencies\")\n\n# Initialize singletons lazily to avoid import-time DB side effects during test collection.\n# Use absolute path relative to this file to ensure plugins are found regardless of CWD\nproject_root = Path(__file__).parent.parent.parent\nconfig_path = project_root / \"config.json\"\n\nconfig_manager: Optional[ConfigManager] = None\nplugin_loader: Optional[PluginLoader] = None\ntask_manager: Optional[TaskManager] = None\nscheduler_service: Optional[SchedulerService] = None\nresource_service: Optional[ResourceService] = None\n\n\n# [DEF:get_config_manager:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for ConfigManager.\n# @PRE: Global config_manager must be initialized.\n# @POST: Returns shared ConfigManager instance.\n# @RETURN: ConfigManager - The shared config manager instance.\ndef get_config_manager() -> ConfigManager:\n \"\"\"Dependency injector for ConfigManager.\"\"\"\n global config_manager\n if config_manager is None:\n init_db()\n config_manager = ConfigManager(config_path=str(config_path))\n return config_manager\n\n\n# [/DEF:get_config_manager:Function]\n\nplugin_dir = Path(__file__).parent / \"plugins\"\n\n# Clean Release Redesign Singletons\n# Note: These use get_db() which is a generator, so we need a way to provide a session.\n# For singletons in dependencies.py, we might need a different approach or\n# initialize them inside the dependency functions.\n\n\n# [DEF:get_plugin_loader:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for PluginLoader.\n# @PRE: Global plugin_loader must be initialized.\n# @POST: Returns shared PluginLoader instance.\n# @RETURN: PluginLoader - The shared plugin loader instance.\ndef get_plugin_loader() -> PluginLoader:\n \"\"\"Dependency injector for PluginLoader.\"\"\"\n global plugin_loader\n if plugin_loader is None:\n plugin_loader = PluginLoader(plugin_dir=str(plugin_dir))\n log.reason(f\"PluginLoader initialized with directory: {plugin_dir}\")\n log.reason(\n f\"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}\"\n )\n return plugin_loader\n\n\n# [/DEF:get_plugin_loader:Function]\n\n\n# [DEF:get_task_manager:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for TaskManager.\n# @PRE: Global task_manager must be initialized.\n# @POST: Returns shared TaskManager instance.\n# @RETURN: TaskManager - The shared task manager instance.\ndef get_task_manager() -> TaskManager:\n \"\"\"Dependency injector for TaskManager.\"\"\"\n global task_manager\n if task_manager is None:\n task_manager = TaskManager(get_plugin_loader())\n log.reason(\"TaskManager initialized\")\n return task_manager\n\n\n# [/DEF:get_task_manager:Function]\n\n\n# [DEF:get_scheduler_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for SchedulerService.\n# @PRE: Global scheduler_service must be initialized.\n# @POST: Returns shared SchedulerService instance.\n# @RETURN: SchedulerService - The shared scheduler service instance.\ndef get_scheduler_service() -> SchedulerService:\n \"\"\"Dependency injector for SchedulerService.\"\"\"\n global scheduler_service\n if scheduler_service is None:\n scheduler_service = SchedulerService(get_task_manager(), get_config_manager())\n log.reason(\"SchedulerService initialized\")\n return scheduler_service\n\n\n# [/DEF:get_scheduler_service:Function]\n\n\n# [DEF:get_resource_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for ResourceService.\n# @PRE: Global resource_service must be initialized.\n# @POST: Returns shared ResourceService instance.\n# @RETURN: ResourceService - The shared resource service instance.\ndef get_resource_service() -> ResourceService:\n \"\"\"Dependency injector for ResourceService.\"\"\"\n global resource_service\n if resource_service is None:\n resource_service = ResourceService()\n log.reason(\"ResourceService initialized\")\n return resource_service\n\n\n# [/DEF:get_resource_service:Function]\n\n\n# [DEF:get_mapping_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for MappingService.\n# @PRE: Global config_manager must be initialized.\n# @POST: Returns new MappingService instance.\n# @RETURN: MappingService - A new mapping service instance.\ndef get_mapping_service() -> MappingService:\n \"\"\"Dependency injector for MappingService.\"\"\"\n return MappingService(get_config_manager())\n\n\n# [/DEF:get_mapping_service:Function]\n\n\n_clean_release_repository = CleanReleaseRepository()\n\n\n# [DEF:get_clean_release_repository:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Legacy compatibility shim for CleanReleaseRepository.\n# @POST: Returns a shared CleanReleaseRepository instance.\ndef get_clean_release_repository() -> CleanReleaseRepository:\n \"\"\"Legacy compatibility shim for CleanReleaseRepository.\"\"\"\n return _clean_release_repository\n\n\n# [/DEF:get_clean_release_repository:Function]\n\n\n# [DEF:get_clean_release_facade:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for CleanReleaseFacade.\n# @POST: Returns a facade instance with a fresh DB session.\ndef get_clean_release_facade(db=Depends(get_db)) -> CleanReleaseFacade:\n candidate_repo = CandidateRepository(db)\n artifact_repo = ArtifactRepository(db)\n manifest_repo = ManifestRepository(db)\n policy_repo = PolicyRepository(db)\n compliance_repo = ComplianceRepository(db)\n report_repo = ReportRepository(db)\n approval_repo = ApprovalRepository(db)\n publication_repo = PublicationRepository(db)\n audit_repo = AuditRepository(db)\n\n return CleanReleaseFacade(\n candidate_repo=candidate_repo,\n artifact_repo=artifact_repo,\n manifest_repo=manifest_repo,\n policy_repo=policy_repo,\n compliance_repo=compliance_repo,\n report_repo=report_repo,\n approval_repo=approval_repo,\n publication_repo=publication_repo,\n audit_repo=audit_repo,\n config_manager=get_config_manager(),\n )\n\n\n# [/DEF:get_clean_release_facade:Function]\n\n# [DEF:oauth2_scheme:Variable]\n# @RELATION: DEPENDS_ON -> OAuth2PasswordBearer\n# @COMPLEXITY: 1\n# @PURPOSE: OAuth2 password bearer scheme for token extraction.\noauth2_scheme = OAuth2PasswordBearer(tokenUrl=\"/api/auth/login\")\n# [/DEF:oauth2_scheme:Variable]\n\n\n# [DEF:get_current_user:Function]\n# @RELATION: CALLS -> AuthRepository\n# @COMPLEXITY: 3\n# @PURPOSE: Dependency for retrieving currently authenticated user from a JWT.\n# @PRE: JWT token provided in Authorization header.\n# @POST: Returns User object if token is valid.\n# @THROW: HTTPException 401 if token is invalid or user not found.\n# @PARAM: token (str) - Extracted JWT token.\n# @PARAM: db (Session) - Auth database session.\n# @RETURN: User - The authenticated user.\ndef get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db)):\n credentials_exception = HTTPException(\n status_code=status.HTTP_401_UNAUTHORIZED,\n detail=\"Could not validate credentials\",\n headers={\"WWW-Authenticate\": \"Bearer\"},\n )\n try:\n payload = decode_token(token)\n username_value = payload.get(\"sub\")\n if not isinstance(username_value, str) or not username_value:\n raise credentials_exception\n username = username_value\n except JWTError:\n raise credentials_exception\n\n repo = AuthRepository(db)\n user = repo.get_user_by_username(username)\n if user is None:\n raise credentials_exception\n return user\n\n\n# [/DEF:get_current_user:Function]\n\n\n# [DEF:has_permission:Function]\n# @RELATION: CALLS -> AuthRepository\n# @COMPLEXITY: 3\n# @PURPOSE: Dependency for checking if the current user has a specific permission.\n# @PRE: User is authenticated.\n# @POST: Returns True if user has permission.\n# @THROW: HTTPException 403 if permission is denied.\n# @PARAM: resource (str) - The resource identifier.\n# @PARAM: action (str) - The action identifier (READ, EXECUTE, WRITE).\n# @RETURN: User - The authenticated user if permission granted.\ndef has_permission(resource: str, action: str):\n def permission_checker(current_user: User = Depends(get_current_user)):\n # Union of all permissions across all roles\n for role in current_user.roles:\n for perm in role.permissions:\n if perm.resource == resource and perm.action == action:\n return current_user\n\n # Special case for Admin role (full access)\n if any(role.name == \"Admin\" for role in current_user.roles):\n return current_user\n\n from .core.auth.logger import log_security_event\n\n log_security_event(\n \"PERMISSION_DENIED\",\n str(getattr(current_user, \"username\", \"unknown\")),\n {\"resource\": resource, \"action\": action},\n )\n\n raise HTTPException(\n status_code=status.HTTP_403_FORBIDDEN,\n detail=f\"Permission denied for {resource}:{action}\",\n )\n\n return permission_checker\n\n\n# [/DEF:has_permission:Function]\n\n# [/DEF:AppDependencies:Module]\n" }, { "contract_id": "get_config_manager", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 61, - "end_line": 76, + "start_line": 63, + "end_line": 78, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52585,8 +53594,8 @@ "contract_id": "get_plugin_loader", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 86, - "end_line": 104, + "start_line": 88, + "end_line": 106, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52624,14 +53633,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_plugin_loader:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for PluginLoader.\n# @PRE: Global plugin_loader must be initialized.\n# @POST: Returns shared PluginLoader instance.\n# @RETURN: PluginLoader - The shared plugin loader instance.\ndef get_plugin_loader() -> PluginLoader:\n \"\"\"Dependency injector for PluginLoader.\"\"\"\n global plugin_loader\n if plugin_loader is None:\n plugin_loader = PluginLoader(plugin_dir=str(plugin_dir))\n logger.info(f\"PluginLoader initialized with directory: {plugin_dir}\")\n logger.info(\n f\"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}\"\n )\n return plugin_loader\n\n\n# [/DEF:get_plugin_loader:Function]\n" + "body": "# [DEF:get_plugin_loader:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for PluginLoader.\n# @PRE: Global plugin_loader must be initialized.\n# @POST: Returns shared PluginLoader instance.\n# @RETURN: PluginLoader - The shared plugin loader instance.\ndef get_plugin_loader() -> PluginLoader:\n \"\"\"Dependency injector for PluginLoader.\"\"\"\n global plugin_loader\n if plugin_loader is None:\n plugin_loader = PluginLoader(plugin_dir=str(plugin_dir))\n log.reason(f\"PluginLoader initialized with directory: {plugin_dir}\")\n log.reason(\n f\"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}\"\n )\n return plugin_loader\n\n\n# [/DEF:get_plugin_loader:Function]\n" }, { "contract_id": "get_task_manager", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 107, - "end_line": 122, + "start_line": 109, + "end_line": 124, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52669,14 +53678,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_task_manager:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for TaskManager.\n# @PRE: Global task_manager must be initialized.\n# @POST: Returns shared TaskManager instance.\n# @RETURN: TaskManager - The shared task manager instance.\ndef get_task_manager() -> TaskManager:\n \"\"\"Dependency injector for TaskManager.\"\"\"\n global task_manager\n if task_manager is None:\n task_manager = TaskManager(get_plugin_loader())\n logger.info(\"TaskManager initialized\")\n return task_manager\n\n\n# [/DEF:get_task_manager:Function]\n" + "body": "# [DEF:get_task_manager:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for TaskManager.\n# @PRE: Global task_manager must be initialized.\n# @POST: Returns shared TaskManager instance.\n# @RETURN: TaskManager - The shared task manager instance.\ndef get_task_manager() -> TaskManager:\n \"\"\"Dependency injector for TaskManager.\"\"\"\n global task_manager\n if task_manager is None:\n task_manager = TaskManager(get_plugin_loader())\n log.reason(\"TaskManager initialized\")\n return task_manager\n\n\n# [/DEF:get_task_manager:Function]\n" }, { "contract_id": "get_scheduler_service", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 125, - "end_line": 140, + "start_line": 127, + "end_line": 142, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52714,14 +53723,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_scheduler_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for SchedulerService.\n# @PRE: Global scheduler_service must be initialized.\n# @POST: Returns shared SchedulerService instance.\n# @RETURN: SchedulerService - The shared scheduler service instance.\ndef get_scheduler_service() -> SchedulerService:\n \"\"\"Dependency injector for SchedulerService.\"\"\"\n global scheduler_service\n if scheduler_service is None:\n scheduler_service = SchedulerService(get_task_manager(), get_config_manager())\n logger.info(\"SchedulerService initialized\")\n return scheduler_service\n\n\n# [/DEF:get_scheduler_service:Function]\n" + "body": "# [DEF:get_scheduler_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for SchedulerService.\n# @PRE: Global scheduler_service must be initialized.\n# @POST: Returns shared SchedulerService instance.\n# @RETURN: SchedulerService - The shared scheduler service instance.\ndef get_scheduler_service() -> SchedulerService:\n \"\"\"Dependency injector for SchedulerService.\"\"\"\n global scheduler_service\n if scheduler_service is None:\n scheduler_service = SchedulerService(get_task_manager(), get_config_manager())\n log.reason(\"SchedulerService initialized\")\n return scheduler_service\n\n\n# [/DEF:get_scheduler_service:Function]\n" }, { "contract_id": "get_resource_service", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 143, - "end_line": 158, + "start_line": 145, + "end_line": 160, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52759,14 +53768,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:get_resource_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for ResourceService.\n# @PRE: Global resource_service must be initialized.\n# @POST: Returns shared ResourceService instance.\n# @RETURN: ResourceService - The shared resource service instance.\ndef get_resource_service() -> ResourceService:\n \"\"\"Dependency injector for ResourceService.\"\"\"\n global resource_service\n if resource_service is None:\n resource_service = ResourceService()\n logger.info(\"ResourceService initialized\")\n return resource_service\n\n\n# [/DEF:get_resource_service:Function]\n" + "body": "# [DEF:get_resource_service:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Dependency injector for ResourceService.\n# @PRE: Global resource_service must be initialized.\n# @POST: Returns shared ResourceService instance.\n# @RETURN: ResourceService - The shared resource service instance.\ndef get_resource_service() -> ResourceService:\n \"\"\"Dependency injector for ResourceService.\"\"\"\n global resource_service\n if resource_service is None:\n resource_service = ResourceService()\n log.reason(\"ResourceService initialized\")\n return resource_service\n\n\n# [/DEF:get_resource_service:Function]\n" }, { "contract_id": "get_mapping_service", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 161, - "end_line": 172, + "start_line": 163, + "end_line": 174, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52810,8 +53819,8 @@ "contract_id": "get_clean_release_repository", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 178, - "end_line": 187, + "start_line": 180, + "end_line": 189, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52838,8 +53847,8 @@ "contract_id": "get_clean_release_facade", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 190, - "end_line": 219, + "start_line": 192, + "end_line": 221, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52866,8 +53875,8 @@ "contract_id": "oauth2_scheme", "contract_type": "Variable", "file_path": "backend/src/dependencies.py", - "start_line": 221, - "end_line": 226, + "start_line": 223, + "end_line": 228, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -52949,8 +53958,8 @@ "contract_id": "get_current_user", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 229, - "end_line": 261, + "start_line": 231, + "end_line": 263, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -53015,8 +54024,8 @@ "contract_id": "has_permission", "contract_type": "Function", "file_path": "backend/src/dependencies.py", - "start_line": 264, - "end_line": 302, + "start_line": 266, + "end_line": 304, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -59445,7 +60454,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/git_plugin.py", "start_line": 1, - "end_line": 401, + "end_line": 404, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -59620,14 +60629,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitPluginModule:Module]\n#\n# @SEMANTICS: git, plugin, dashboard, version_control, sync, deploy\n# @PURPOSE: Предоставляет плагин для версионирования и развертывания дашбордов Superset.\n# @LAYER: Plugin\n# @RELATION: INHERITS_FROM -> src.core.plugin_base.PluginBase\n# @RELATION: USES -> src.services.git_service.GitService\n# @RELATION: USES -> src.core.superset_client.SupersetClient\n# @RELATION: USES -> src.core.config_manager.ConfigManager\n# @RELATION: USES -> TaskContext\n#\n# @INVARIANT: Все операции с Git должны выполняться через GitService.\n# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.\n\n# [SECTION: IMPORTS]\nimport os\nimport io\nimport shutil\nimport zipfile\nfrom pathlib import Path\nfrom typing import Dict, Any, Optional\nfrom src.core.plugin_base import PluginBase\nfrom src.services.git_service import GitService\nfrom src.core.logger import logger as app_logger, belief_scope\nfrom src.core.config_manager import ConfigManager\nfrom src.core.superset_client import SupersetClient\nfrom src.core.task_manager.context import TaskContext\nfrom src.core.database import SessionLocal\nfrom src.core.mapping_service import IdMappingService\n# [/SECTION]\n\n# [DEF:GitPlugin:Class]\n# @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов.\nclass GitPlugin(PluginBase):\n \n # [DEF:__init__:Function]\n # @PURPOSE: Инициализирует плагин и его зависимости.\n # @PRE: config.json exists or shared config_manager is available.\n # @POST: Инициализированы git_service и config_manager.\n def __init__(self):\n with belief_scope(\"GitPlugin.__init__\"):\n app_logger.info(\"Initializing GitPlugin.\")\n self.git_service = GitService()\n \n # Robust config path resolution:\n # 1. Try absolute path from src/dependencies.py style if possible\n # 2. Try relative paths based on common execution patterns\n if os.path.exists(\"../config.json\"):\n config_path = \"../config.json\"\n elif os.path.exists(\"config.json\"):\n config_path = \"config.json\"\n else:\n # Fallback to the one initialized in dependencies if we can import it\n try:\n from src.dependencies import config_manager\n self.config_manager = config_manager\n app_logger.info(\"GitPlugin initialized using shared config_manager.\")\n return\n except Exception:\n config_path = \"config.json\"\n\n self.config_manager = ConfigManager(config_path)\n app_logger.info(f\"GitPlugin initialized with {config_path}\")\n # [/DEF:__init__:Function]\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the plugin identifier.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns 'git-integration'.\n def id(self) -> str:\n with belief_scope(\"GitPlugin.id\"):\n return \"git-integration\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the plugin name.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the human-readable name.\n def name(self) -> str:\n with belief_scope(\"GitPlugin.name\"):\n return \"Git Integration\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns the plugin description.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the plugin's purpose description.\n def description(self) -> str:\n with belief_scope(\"GitPlugin.description\"):\n return \"Version control for Superset dashboards\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the plugin version.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the version string.\n def version(self) -> str:\n with belief_scope(\"GitPlugin.version\"):\n return \"0.1.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend route for the git plugin.\n # @RETURN: str - \"/git\"\n def ui_route(self) -> str:\n with belief_scope(\"GitPlugin.ui_route\"):\n return \"/git\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns a JSON schema dictionary.\n # @RETURN: Dict[str, Any] - Схема параметров.\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.get_schema\"):\n return {\n \"type\": \"object\",\n \"properties\": {\n \"operation\": {\"type\": \"string\", \"enum\": [\"sync\", \"deploy\", \"history\"]},\n \"dashboard_id\": {\"type\": \"integer\"},\n \"environment_id\": {\"type\": \"string\"},\n \"source_env_id\": {\"type\": \"string\"}\n },\n \"required\": [\"operation\", \"dashboard_id\"]\n }\n # [/DEF:get_schema:Function]\n\n # [DEF:initialize:Function]\n # @PURPOSE: Выполняет начальную настройку плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Плагин готов к выполнению задач.\n async def initialize(self):\n with belief_scope(\"GitPlugin.initialize\"):\n app_logger.info(\"[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.\")\n\n # [DEF:execute:Function]\n # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.\n # @PRE: task_data содержит 'operation' и 'dashboard_id'.\n # @POST: Возвращает результат выполнения операции.\n # @PARAM: task_data (Dict[str, Any]) - Данные задачи.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @RETURN: Dict[str, Any] - Статус и сообщение.\n # @RELATION: CALLS -> self._handle_sync\n # @RELATION: CALLS -> self._handle_deploy\n async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.execute\"):\n operation = task_data.get(\"operation\")\n dashboard_id = task_data.get(\"dashboard_id\")\n \n # Use TaskContext logger if available, otherwise fall back to app_logger\n log = context.logger if context else app_logger\n \n # Create sub-loggers for different components\n git_log = log.with_source(\"git\") if context else log\n superset_log = log.with_source(\"superset_api\") if context else log\n \n log.info(f\"Executing operation: {operation} for dashboard {dashboard_id}\")\n \n if operation == \"sync\":\n source_env_id = task_data.get(\"source_env_id\")\n result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)\n elif operation == \"deploy\":\n env_id = task_data.get(\"environment_id\")\n result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)\n elif operation == \"history\":\n result = {\"status\": \"success\", \"message\": \"History available via API\"}\n else:\n log.error(f\"Unknown operation: {operation}\")\n raise ValueError(f\"Unknown operation: {operation}\")\n \n log.info(f\"Operation {operation} completed.\")\n return result\n # [/DEF:execute:Function]\n\n # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n app_logger.info(\"[_handle_sync][Action] Changes staged in git\")\n except Exception as ge:\n app_logger.warning(f\"[_handle_sync][Action] Failed to stage changes: {ge}\")\n\n app_logger.info(f\"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.\")\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n app_logger.error(f\"[_handle_sync][Coherence:Failed] Sync failed: {e}\")\n raise\n # [/DEF:_handle_sync:Function]\n\n # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n app_logger.info(f\"[_handle_deploy][Action] Importing dashboard to {env.name}\")\n result = client.import_dashboard(temp_zip_path)\n app_logger.info(f\"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.\")\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n app_logger.error(f\"[_handle_deploy][Coherence:Failed] Deployment failed: {e}\")\n raise\n # [/DEF:_handle_deploy:Function]\n\n # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n app_logger.info(f\"[_get_env][Entry] Fetching environment for ID: {env_id}\")\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}\")\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n app_logger.info(f\"[_get_env][Exit] Found environment in DB: {db_env.name}\")\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment {env_id} in ConfigManager list\")\n return env\n \n if not env_id:\n app_logger.info(f\"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}\")\n return envs[0]\n \n app_logger.error(f\"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n\n # [/DEF:initialize:Function]\n# [/DEF:GitPlugin:Class]\n# [/DEF:GitPluginModule:Module]\n" + "body": "# [DEF:GitPluginModule:Module]\n#\n# @SEMANTICS: git, plugin, dashboard, version_control, sync, deploy\n# @PURPOSE: Предоставляет плагин для версионирования и развертывания дашбордов Superset.\n# @LAYER: Plugin\n# @RELATION: INHERITS_FROM -> src.core.plugin_base.PluginBase\n# @RELATION: USES -> src.services.git_service.GitService\n# @RELATION: USES -> src.core.superset_client.SupersetClient\n# @RELATION: USES -> src.core.config_manager.ConfigManager\n# @RELATION: USES -> TaskContext\n#\n# @INVARIANT: Все операции с Git должны выполняться через GitService.\n# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.\n\n# [SECTION: IMPORTS]\nimport os\nimport io\nimport shutil\nimport zipfile\nfrom pathlib import Path\nfrom typing import Dict, Any, Optional\nfrom src.core.plugin_base import PluginBase\nfrom src.services.git_service import GitService\nfrom src.core.logger import logger as app_logger, belief_scope\nfrom src.core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"GitPlugin\")\nfrom src.core.config_manager import ConfigManager\nfrom src.core.superset_client import SupersetClient\nfrom src.core.task_manager.context import TaskContext\nfrom src.core.database import SessionLocal\nfrom src.core.mapping_service import IdMappingService\n# [/SECTION]\n\n# [DEF:GitPlugin:Class]\n# @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов.\nclass GitPlugin(PluginBase):\n \n # [DEF:__init__:Function]\n # @PURPOSE: Инициализирует плагин и его зависимости.\n # @PRE: config.json exists or shared config_manager is available.\n # @POST: Инициализированы git_service и config_manager.\n def __init__(self):\n with belief_scope(\"GitPlugin.__init__\"):\n log.reason(\"Initializing GitPlugin.\")\n self.git_service = GitService()\n \n # Robust config path resolution:\n # 1. Try absolute path from src/dependencies.py style if possible\n # 2. Try relative paths based on common execution patterns\n if os.path.exists(\"../config.json\"):\n config_path = \"../config.json\"\n elif os.path.exists(\"config.json\"):\n config_path = \"config.json\"\n else:\n # Fallback to the one initialized in dependencies if we can import it\n try:\n from src.dependencies import config_manager\n self.config_manager = config_manager\n log.reflect(\"GitPlugin initialized using shared config_manager.\")\n return\n except Exception:\n config_path = \"config.json\"\n\n self.config_manager = ConfigManager(config_path)\n log.reflect(f\"GitPlugin initialized with {config_path}\")\n # [/DEF:__init__:Function]\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the plugin identifier.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns 'git-integration'.\n def id(self) -> str:\n with belief_scope(\"GitPlugin.id\"):\n return \"git-integration\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the plugin name.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the human-readable name.\n def name(self) -> str:\n with belief_scope(\"GitPlugin.name\"):\n return \"Git Integration\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns the plugin description.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the plugin's purpose description.\n def description(self) -> str:\n with belief_scope(\"GitPlugin.description\"):\n return \"Version control for Superset dashboards\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the plugin version.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the version string.\n def version(self) -> str:\n with belief_scope(\"GitPlugin.version\"):\n return \"0.1.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend route for the git plugin.\n # @RETURN: str - \"/git\"\n def ui_route(self) -> str:\n with belief_scope(\"GitPlugin.ui_route\"):\n return \"/git\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns a JSON schema dictionary.\n # @RETURN: Dict[str, Any] - Схема параметров.\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.get_schema\"):\n return {\n \"type\": \"object\",\n \"properties\": {\n \"operation\": {\"type\": \"string\", \"enum\": [\"sync\", \"deploy\", \"history\"]},\n \"dashboard_id\": {\"type\": \"integer\"},\n \"environment_id\": {\"type\": \"string\"},\n \"source_env_id\": {\"type\": \"string\"}\n },\n \"required\": [\"operation\", \"dashboard_id\"]\n }\n # [/DEF:get_schema:Function]\n\n # [DEF:initialize:Function]\n # @PURPOSE: Выполняет начальную настройку плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Плагин готов к выполнению задач.\n async def initialize(self):\n with belief_scope(\"GitPlugin.initialize\"):\n log.reason(\"Initializing Git Integration Plugin logic\")\n\n # [DEF:execute:Function]\n # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.\n # @PRE: task_data содержит 'operation' и 'dashboard_id'.\n # @POST: Возвращает результат выполнения операции.\n # @PARAM: task_data (Dict[str, Any]) - Данные задачи.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @RETURN: Dict[str, Any] - Статус и сообщение.\n # @RELATION: CALLS -> self._handle_sync\n # @RELATION: CALLS -> self._handle_deploy\n async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.execute\"):\n operation = task_data.get(\"operation\")\n dashboard_id = task_data.get(\"dashboard_id\")\n \n # Use TaskContext logger if available, otherwise fall back to app_logger\n log = context.logger if context else app_logger\n \n # Create sub-loggers for different components\n git_log = log.with_source(\"git\") if context else log\n superset_log = log.with_source(\"superset_api\") if context else log\n \n log.info(f\"Executing operation: {operation} for dashboard {dashboard_id}\")\n \n if operation == \"sync\":\n source_env_id = task_data.get(\"source_env_id\")\n result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)\n elif operation == \"deploy\":\n env_id = task_data.get(\"environment_id\")\n result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)\n elif operation == \"history\":\n result = {\"status\": \"success\", \"message\": \"History available via API\"}\n else:\n log.error(f\"Unknown operation: {operation}\")\n raise ValueError(f\"Unknown operation: {operation}\")\n \n log.info(f\"Operation {operation} completed.\")\n return result\n # [/DEF:execute:Function]\n\n # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n log.reason(\"Changes staged in git\")\n except Exception as ge:\n log.explore(\"Failed to stage changes\", error=str(ge))\n\n log.reflect(\"Dashboard synced successfully\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n log.explore(\"Sync failed\", error=str(e))\n raise\n # [/DEF:_handle_sync:Function]\n\n # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n log.reason(\"Importing dashboard\", payload={\"environment\": env.name})\n result = client.import_dashboard(temp_zip_path)\n log.reflect(\"Deployment successful\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n log.explore(\"Deployment failed\", error=str(e))\n raise\n # [/DEF:_handle_deploy:Function]\n\n # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n log.reason(\"Fetching environment\", payload={\"env_id\": env_id})\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n log.reflect(\"Found environment by ID in ConfigManager\", payload={\"name\": env.name})\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n log.reflect(\"Found environment in DB\", payload={\"name\": db_env.name})\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n log.reflect(\"Found environment in ConfigManager list\", payload={\"env_id\": env_id})\n return env\n \n if not env_id:\n log.reflect(\"Using first environment from ConfigManager\", payload={\"name\": envs[0].name})\n return envs[0]\n \n log.explore(\"No environments configured\", payload={\"env_id\": env_id}, error=\"No Superset environments found in config or database\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n\n # [/DEF:initialize:Function]\n# [/DEF:GitPlugin:Class]\n# [/DEF:GitPluginModule:Module]\n" }, { "contract_id": "GitPlugin", "contract_type": "Class", "file_path": "backend/src/plugins/git_plugin.py", - "start_line": 32, - "end_line": 400, + "start_line": 35, + "end_line": 403, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -59636,14 +60645,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:GitPlugin:Class]\n# @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов.\nclass GitPlugin(PluginBase):\n \n # [DEF:__init__:Function]\n # @PURPOSE: Инициализирует плагин и его зависимости.\n # @PRE: config.json exists or shared config_manager is available.\n # @POST: Инициализированы git_service и config_manager.\n def __init__(self):\n with belief_scope(\"GitPlugin.__init__\"):\n app_logger.info(\"Initializing GitPlugin.\")\n self.git_service = GitService()\n \n # Robust config path resolution:\n # 1. Try absolute path from src/dependencies.py style if possible\n # 2. Try relative paths based on common execution patterns\n if os.path.exists(\"../config.json\"):\n config_path = \"../config.json\"\n elif os.path.exists(\"config.json\"):\n config_path = \"config.json\"\n else:\n # Fallback to the one initialized in dependencies if we can import it\n try:\n from src.dependencies import config_manager\n self.config_manager = config_manager\n app_logger.info(\"GitPlugin initialized using shared config_manager.\")\n return\n except Exception:\n config_path = \"config.json\"\n\n self.config_manager = ConfigManager(config_path)\n app_logger.info(f\"GitPlugin initialized with {config_path}\")\n # [/DEF:__init__:Function]\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the plugin identifier.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns 'git-integration'.\n def id(self) -> str:\n with belief_scope(\"GitPlugin.id\"):\n return \"git-integration\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the plugin name.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the human-readable name.\n def name(self) -> str:\n with belief_scope(\"GitPlugin.name\"):\n return \"Git Integration\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns the plugin description.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the plugin's purpose description.\n def description(self) -> str:\n with belief_scope(\"GitPlugin.description\"):\n return \"Version control for Superset dashboards\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the plugin version.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the version string.\n def version(self) -> str:\n with belief_scope(\"GitPlugin.version\"):\n return \"0.1.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend route for the git plugin.\n # @RETURN: str - \"/git\"\n def ui_route(self) -> str:\n with belief_scope(\"GitPlugin.ui_route\"):\n return \"/git\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns a JSON schema dictionary.\n # @RETURN: Dict[str, Any] - Схема параметров.\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.get_schema\"):\n return {\n \"type\": \"object\",\n \"properties\": {\n \"operation\": {\"type\": \"string\", \"enum\": [\"sync\", \"deploy\", \"history\"]},\n \"dashboard_id\": {\"type\": \"integer\"},\n \"environment_id\": {\"type\": \"string\"},\n \"source_env_id\": {\"type\": \"string\"}\n },\n \"required\": [\"operation\", \"dashboard_id\"]\n }\n # [/DEF:get_schema:Function]\n\n # [DEF:initialize:Function]\n # @PURPOSE: Выполняет начальную настройку плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Плагин готов к выполнению задач.\n async def initialize(self):\n with belief_scope(\"GitPlugin.initialize\"):\n app_logger.info(\"[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.\")\n\n # [DEF:execute:Function]\n # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.\n # @PRE: task_data содержит 'operation' и 'dashboard_id'.\n # @POST: Возвращает результат выполнения операции.\n # @PARAM: task_data (Dict[str, Any]) - Данные задачи.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @RETURN: Dict[str, Any] - Статус и сообщение.\n # @RELATION: CALLS -> self._handle_sync\n # @RELATION: CALLS -> self._handle_deploy\n async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.execute\"):\n operation = task_data.get(\"operation\")\n dashboard_id = task_data.get(\"dashboard_id\")\n \n # Use TaskContext logger if available, otherwise fall back to app_logger\n log = context.logger if context else app_logger\n \n # Create sub-loggers for different components\n git_log = log.with_source(\"git\") if context else log\n superset_log = log.with_source(\"superset_api\") if context else log\n \n log.info(f\"Executing operation: {operation} for dashboard {dashboard_id}\")\n \n if operation == \"sync\":\n source_env_id = task_data.get(\"source_env_id\")\n result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)\n elif operation == \"deploy\":\n env_id = task_data.get(\"environment_id\")\n result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)\n elif operation == \"history\":\n result = {\"status\": \"success\", \"message\": \"History available via API\"}\n else:\n log.error(f\"Unknown operation: {operation}\")\n raise ValueError(f\"Unknown operation: {operation}\")\n \n log.info(f\"Operation {operation} completed.\")\n return result\n # [/DEF:execute:Function]\n\n # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n app_logger.info(\"[_handle_sync][Action] Changes staged in git\")\n except Exception as ge:\n app_logger.warning(f\"[_handle_sync][Action] Failed to stage changes: {ge}\")\n\n app_logger.info(f\"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.\")\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n app_logger.error(f\"[_handle_sync][Coherence:Failed] Sync failed: {e}\")\n raise\n # [/DEF:_handle_sync:Function]\n\n # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n app_logger.info(f\"[_handle_deploy][Action] Importing dashboard to {env.name}\")\n result = client.import_dashboard(temp_zip_path)\n app_logger.info(f\"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.\")\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n app_logger.error(f\"[_handle_deploy][Coherence:Failed] Deployment failed: {e}\")\n raise\n # [/DEF:_handle_deploy:Function]\n\n # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n app_logger.info(f\"[_get_env][Entry] Fetching environment for ID: {env_id}\")\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}\")\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n app_logger.info(f\"[_get_env][Exit] Found environment in DB: {db_env.name}\")\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment {env_id} in ConfigManager list\")\n return env\n \n if not env_id:\n app_logger.info(f\"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}\")\n return envs[0]\n \n app_logger.error(f\"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n\n # [/DEF:initialize:Function]\n# [/DEF:GitPlugin:Class]\n" + "body": "# [DEF:GitPlugin:Class]\n# @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов.\nclass GitPlugin(PluginBase):\n \n # [DEF:__init__:Function]\n # @PURPOSE: Инициализирует плагин и его зависимости.\n # @PRE: config.json exists or shared config_manager is available.\n # @POST: Инициализированы git_service и config_manager.\n def __init__(self):\n with belief_scope(\"GitPlugin.__init__\"):\n log.reason(\"Initializing GitPlugin.\")\n self.git_service = GitService()\n \n # Robust config path resolution:\n # 1. Try absolute path from src/dependencies.py style if possible\n # 2. Try relative paths based on common execution patterns\n if os.path.exists(\"../config.json\"):\n config_path = \"../config.json\"\n elif os.path.exists(\"config.json\"):\n config_path = \"config.json\"\n else:\n # Fallback to the one initialized in dependencies if we can import it\n try:\n from src.dependencies import config_manager\n self.config_manager = config_manager\n log.reflect(\"GitPlugin initialized using shared config_manager.\")\n return\n except Exception:\n config_path = \"config.json\"\n\n self.config_manager = ConfigManager(config_path)\n log.reflect(f\"GitPlugin initialized with {config_path}\")\n # [/DEF:__init__:Function]\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the plugin identifier.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns 'git-integration'.\n def id(self) -> str:\n with belief_scope(\"GitPlugin.id\"):\n return \"git-integration\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the plugin name.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the human-readable name.\n def name(self) -> str:\n with belief_scope(\"GitPlugin.name\"):\n return \"Git Integration\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns the plugin description.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the plugin's purpose description.\n def description(self) -> str:\n with belief_scope(\"GitPlugin.description\"):\n return \"Version control for Superset dashboards\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the plugin version.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns the version string.\n def version(self) -> str:\n with belief_scope(\"GitPlugin.version\"):\n return \"0.1.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend route for the git plugin.\n # @RETURN: str - \"/git\"\n def ui_route(self) -> str:\n with belief_scope(\"GitPlugin.ui_route\"):\n return \"/git\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Returns a JSON schema dictionary.\n # @RETURN: Dict[str, Any] - Схема параметров.\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.get_schema\"):\n return {\n \"type\": \"object\",\n \"properties\": {\n \"operation\": {\"type\": \"string\", \"enum\": [\"sync\", \"deploy\", \"history\"]},\n \"dashboard_id\": {\"type\": \"integer\"},\n \"environment_id\": {\"type\": \"string\"},\n \"source_env_id\": {\"type\": \"string\"}\n },\n \"required\": [\"operation\", \"dashboard_id\"]\n }\n # [/DEF:get_schema:Function]\n\n # [DEF:initialize:Function]\n # @PURPOSE: Выполняет начальную настройку плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Плагин готов к выполнению задач.\n async def initialize(self):\n with belief_scope(\"GitPlugin.initialize\"):\n log.reason(\"Initializing Git Integration Plugin logic\")\n\n # [DEF:execute:Function]\n # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.\n # @PRE: task_data содержит 'operation' и 'dashboard_id'.\n # @POST: Возвращает результат выполнения операции.\n # @PARAM: task_data (Dict[str, Any]) - Данные задачи.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @RETURN: Dict[str, Any] - Статус и сообщение.\n # @RELATION: CALLS -> self._handle_sync\n # @RELATION: CALLS -> self._handle_deploy\n async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.execute\"):\n operation = task_data.get(\"operation\")\n dashboard_id = task_data.get(\"dashboard_id\")\n \n # Use TaskContext logger if available, otherwise fall back to app_logger\n log = context.logger if context else app_logger\n \n # Create sub-loggers for different components\n git_log = log.with_source(\"git\") if context else log\n superset_log = log.with_source(\"superset_api\") if context else log\n \n log.info(f\"Executing operation: {operation} for dashboard {dashboard_id}\")\n \n if operation == \"sync\":\n source_env_id = task_data.get(\"source_env_id\")\n result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)\n elif operation == \"deploy\":\n env_id = task_data.get(\"environment_id\")\n result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)\n elif operation == \"history\":\n result = {\"status\": \"success\", \"message\": \"History available via API\"}\n else:\n log.error(f\"Unknown operation: {operation}\")\n raise ValueError(f\"Unknown operation: {operation}\")\n \n log.info(f\"Operation {operation} completed.\")\n return result\n # [/DEF:execute:Function]\n\n # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n log.reason(\"Changes staged in git\")\n except Exception as ge:\n log.explore(\"Failed to stage changes\", error=str(ge))\n\n log.reflect(\"Dashboard synced successfully\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n log.explore(\"Sync failed\", error=str(e))\n raise\n # [/DEF:_handle_sync:Function]\n\n # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n log.reason(\"Importing dashboard\", payload={\"environment\": env.name})\n result = client.import_dashboard(temp_zip_path)\n log.reflect(\"Deployment successful\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n log.explore(\"Deployment failed\", error=str(e))\n raise\n # [/DEF:_handle_deploy:Function]\n\n # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n log.reason(\"Fetching environment\", payload={\"env_id\": env_id})\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n log.reflect(\"Found environment by ID in ConfigManager\", payload={\"name\": env.name})\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n log.reflect(\"Found environment in DB\", payload={\"name\": db_env.name})\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n log.reflect(\"Found environment in ConfigManager list\", payload={\"env_id\": env_id})\n return env\n \n if not env_id:\n log.reflect(\"Using first environment from ConfigManager\", payload={\"name\": envs[0].name})\n return envs[0]\n \n log.explore(\"No environments configured\", payload={\"env_id\": env_id}, error=\"No Superset environments found in config or database\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n\n # [/DEF:initialize:Function]\n# [/DEF:GitPlugin:Class]\n" }, { "contract_id": "initialize", "contract_type": "Function", "file_path": "backend/src/plugins/git_plugin.py", - "start_line": 134, - "end_line": 399, + "start_line": 137, + "end_line": 402, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -59673,14 +60682,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:initialize:Function]\n # @PURPOSE: Выполняет начальную настройку плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Плагин готов к выполнению задач.\n async def initialize(self):\n with belief_scope(\"GitPlugin.initialize\"):\n app_logger.info(\"[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.\")\n\n # [DEF:execute:Function]\n # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.\n # @PRE: task_data содержит 'operation' и 'dashboard_id'.\n # @POST: Возвращает результат выполнения операции.\n # @PARAM: task_data (Dict[str, Any]) - Данные задачи.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @RETURN: Dict[str, Any] - Статус и сообщение.\n # @RELATION: CALLS -> self._handle_sync\n # @RELATION: CALLS -> self._handle_deploy\n async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.execute\"):\n operation = task_data.get(\"operation\")\n dashboard_id = task_data.get(\"dashboard_id\")\n \n # Use TaskContext logger if available, otherwise fall back to app_logger\n log = context.logger if context else app_logger\n \n # Create sub-loggers for different components\n git_log = log.with_source(\"git\") if context else log\n superset_log = log.with_source(\"superset_api\") if context else log\n \n log.info(f\"Executing operation: {operation} for dashboard {dashboard_id}\")\n \n if operation == \"sync\":\n source_env_id = task_data.get(\"source_env_id\")\n result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)\n elif operation == \"deploy\":\n env_id = task_data.get(\"environment_id\")\n result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)\n elif operation == \"history\":\n result = {\"status\": \"success\", \"message\": \"History available via API\"}\n else:\n log.error(f\"Unknown operation: {operation}\")\n raise ValueError(f\"Unknown operation: {operation}\")\n \n log.info(f\"Operation {operation} completed.\")\n return result\n # [/DEF:execute:Function]\n\n # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n app_logger.info(\"[_handle_sync][Action] Changes staged in git\")\n except Exception as ge:\n app_logger.warning(f\"[_handle_sync][Action] Failed to stage changes: {ge}\")\n\n app_logger.info(f\"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.\")\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n app_logger.error(f\"[_handle_sync][Coherence:Failed] Sync failed: {e}\")\n raise\n # [/DEF:_handle_sync:Function]\n\n # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n app_logger.info(f\"[_handle_deploy][Action] Importing dashboard to {env.name}\")\n result = client.import_dashboard(temp_zip_path)\n app_logger.info(f\"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.\")\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n app_logger.error(f\"[_handle_deploy][Coherence:Failed] Deployment failed: {e}\")\n raise\n # [/DEF:_handle_deploy:Function]\n\n # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n app_logger.info(f\"[_get_env][Entry] Fetching environment for ID: {env_id}\")\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}\")\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n app_logger.info(f\"[_get_env][Exit] Found environment in DB: {db_env.name}\")\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment {env_id} in ConfigManager list\")\n return env\n \n if not env_id:\n app_logger.info(f\"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}\")\n return envs[0]\n \n app_logger.error(f\"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n\n # [/DEF:initialize:Function]\n" + "body": " # [DEF:initialize:Function]\n # @PURPOSE: Выполняет начальную настройку плагина.\n # @PRE: GitPlugin is initialized.\n # @POST: Плагин готов к выполнению задач.\n async def initialize(self):\n with belief_scope(\"GitPlugin.initialize\"):\n log.reason(\"Initializing Git Integration Plugin logic\")\n\n # [DEF:execute:Function]\n # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.\n # @PRE: task_data содержит 'operation' и 'dashboard_id'.\n # @POST: Возвращает результат выполнения операции.\n # @PARAM: task_data (Dict[str, Any]) - Данные задачи.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @RETURN: Dict[str, Any] - Статус и сообщение.\n # @RELATION: CALLS -> self._handle_sync\n # @RELATION: CALLS -> self._handle_deploy\n async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin.execute\"):\n operation = task_data.get(\"operation\")\n dashboard_id = task_data.get(\"dashboard_id\")\n \n # Use TaskContext logger if available, otherwise fall back to app_logger\n log = context.logger if context else app_logger\n \n # Create sub-loggers for different components\n git_log = log.with_source(\"git\") if context else log\n superset_log = log.with_source(\"superset_api\") if context else log\n \n log.info(f\"Executing operation: {operation} for dashboard {dashboard_id}\")\n \n if operation == \"sync\":\n source_env_id = task_data.get(\"source_env_id\")\n result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)\n elif operation == \"deploy\":\n env_id = task_data.get(\"environment_id\")\n result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)\n elif operation == \"history\":\n result = {\"status\": \"success\", \"message\": \"History available via API\"}\n else:\n log.error(f\"Unknown operation: {operation}\")\n raise ValueError(f\"Unknown operation: {operation}\")\n \n log.info(f\"Operation {operation} completed.\")\n return result\n # [/DEF:execute:Function]\n\n # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n log.reason(\"Changes staged in git\")\n except Exception as ge:\n log.explore(\"Failed to stage changes\", error=str(ge))\n\n log.reflect(\"Dashboard synced successfully\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n log.explore(\"Sync failed\", error=str(e))\n raise\n # [/DEF:_handle_sync:Function]\n\n # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n log.reason(\"Importing dashboard\", payload={\"environment\": env.name})\n result = client.import_dashboard(temp_zip_path)\n log.reflect(\"Deployment successful\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n log.explore(\"Deployment failed\", error=str(e))\n raise\n # [/DEF:_handle_deploy:Function]\n\n # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n log.reason(\"Fetching environment\", payload={\"env_id\": env_id})\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n log.reflect(\"Found environment by ID in ConfigManager\", payload={\"name\": env.name})\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n log.reflect(\"Found environment in DB\", payload={\"name\": db_env.name})\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n log.reflect(\"Found environment in ConfigManager list\", payload={\"env_id\": env_id})\n return env\n \n if not env_id:\n log.reflect(\"Using first environment from ConfigManager\", payload={\"name\": envs[0].name})\n return envs[0]\n \n log.explore(\"No environments configured\", payload={\"env_id\": env_id}, error=\"No Superset environments found in config or database\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n\n # [/DEF:initialize:Function]\n" }, { "contract_id": "_handle_sync", "contract_type": "Function", "file_path": "backend/src/plugins/git_plugin.py", - "start_line": 181, - "end_line": 261, + "start_line": 184, + "end_line": 264, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -59756,14 +60765,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n app_logger.info(\"[_handle_sync][Action] Changes staged in git\")\n except Exception as ge:\n app_logger.warning(f\"[_handle_sync][Action] Failed to stage changes: {ge}\")\n\n app_logger.info(f\"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.\")\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n app_logger.error(f\"[_handle_sync][Coherence:Failed] Sync failed: {e}\")\n raise\n # [/DEF:_handle_sync:Function]\n" + "body": " # [DEF:_handle_sync:Function]\n # @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.\n # @PRE: Репозиторий для дашборда должен существовать.\n # @POST: Файлы в репозитории обновлены до текущего состояния в Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: source_env_id (Optional[str]) - ID исходного окружения.\n # @RETURN: Dict[str, str] - Результат синхронизации.\n # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.\n # @RELATION: CALLS -> src.services.git_service.GitService.get_repo\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard\n async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:\n with belief_scope(\"GitPlugin._handle_sync\"):\n try:\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n git_log.info(f\"Target repo path: {repo_path}\")\n\n # 2. Настройка клиента Superset\n env = self._get_env(source_env_id)\n client = SupersetClient(env)\n client.authenticate()\n\n # 3. Экспорт дашборда\n superset_log.info(f\"Exporting dashboard {dashboard_id} from {env.name}\")\n zip_bytes, _ = client.export_dashboard(dashboard_id)\n \n # 4. Распаковка с выравниванием структуры (flattening)\n git_log.info(f\"Unpacking export to {repo_path}\")\n \n # Список папок/файлов, которые мы ожидаем от Superset\n managed_dirs = [\"dashboards\", \"charts\", \"datasets\", \"databases\"]\n managed_files = [\"metadata.yaml\"]\n \n # Очистка старых данных перед распаковкой, чтобы не оставалось \"призраков\"\n for d in managed_dirs:\n d_path = repo_path / d\n if d_path.exists() and d_path.is_dir():\n shutil.rmtree(d_path)\n for f in managed_files:\n f_path = repo_path / f\n if f_path.exists():\n f_path.unlink()\n\n with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:\n # Superset экспортирует всё в подпапку dashboard_export_timestamp/\n # Нам нужно найти это имя папки\n namelist = zf.namelist()\n if not namelist:\n raise ValueError(\"Export ZIP is empty\")\n \n root_folder = namelist[0].split('/')[0]\n git_log.info(f\"Detected root folder in ZIP: {root_folder}\")\n \n for member in zf.infolist():\n if member.filename.startswith(root_folder + \"/\") and len(member.filename) > len(root_folder) + 1:\n # Убираем префикс папки\n relative_path = member.filename[len(root_folder)+1:]\n target_path = repo_path / relative_path\n \n if member.is_dir():\n target_path.mkdir(parents=True, exist_ok=True)\n else:\n target_path.parent.mkdir(parents=True, exist_ok=True)\n with zf.open(member) as source, open(target_path, \"wb\") as target:\n shutil.copyfileobj(source, target)\n \n # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)\n try:\n repo.git.add(A=True)\n log.reason(\"Changes staged in git\")\n except Exception as ge:\n log.explore(\"Failed to stage changes\", error=str(ge))\n\n log.reflect(\"Dashboard synced successfully\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": \"Dashboard synced and flattened in local repository\"}\n\n except Exception as e:\n log.explore(\"Sync failed\", error=str(e))\n raise\n # [/DEF:_handle_sync:Function]\n" }, { "contract_id": "_handle_deploy", "contract_type": "Function", "file_path": "backend/src/plugins/git_plugin.py", - "start_line": 263, - "end_line": 332, + "start_line": 266, + "end_line": 335, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -59833,14 +60842,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n app_logger.info(f\"[_handle_deploy][Action] Importing dashboard to {env.name}\")\n result = client.import_dashboard(temp_zip_path)\n app_logger.info(f\"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.\")\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n app_logger.error(f\"[_handle_deploy][Coherence:Failed] Deployment failed: {e}\")\n raise\n # [/DEF:_handle_deploy:Function]\n" + "body": " # [DEF:_handle_deploy:Function]\n # @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.\n # @PRE: environment_id должен соответствовать настроенному окружению.\n # @POST: Дашборд импортирован в целевой Superset.\n # @PARAM: dashboard_id (int) - ID дашборда.\n # @PARAM: env_id (str) - ID целевого окружения.\n # @PARAM: log - Main logger instance.\n # @PARAM: git_log - Git-specific logger instance.\n # @PARAM: superset_log - Superset API-specific logger instance.\n # @RETURN: Dict[str, Any] - Результат деплоя.\n # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.\n # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard\n async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:\n with belief_scope(\"GitPlugin._handle_deploy\"):\n try:\n if not env_id:\n raise ValueError(\"Target environment ID required for deployment\")\n\n # 1. Получение репозитория\n repo = self.git_service.get_repo(dashboard_id)\n repo_path = Path(repo.working_dir)\n\n # 2. Упаковка в ZIP\n git_log.info(f\"Packing repository {repo_path} for deployment.\")\n zip_buffer = io.BytesIO()\n \n # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)\n root_dir_name = f\"dashboard_export_{dashboard_id}\"\n \n with zipfile.ZipFile(zip_buffer, \"w\", zipfile.ZIP_DEFLATED) as zf:\n for root, dirs, files in os.walk(repo_path):\n if \".git\" in dirs:\n dirs.remove(\".git\")\n for file in files:\n if file == \".git\" or file.endswith(\".zip\"):\n continue\n file_path = Path(root) / file\n # Prepend the root directory name to the archive path\n arcname = Path(root_dir_name) / file_path.relative_to(repo_path)\n zf.write(file_path, arcname)\n \n zip_buffer.seek(0)\n \n # 3. Настройка клиента Superset\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment {env_id} not found\")\n \n client = SupersetClient(env)\n client.authenticate()\n\n # 4. Импорт\n temp_zip_path = repo_path / f\"deploy_{dashboard_id}.zip\"\n git_log.info(f\"Saving temporary zip to {temp_zip_path}\")\n with open(temp_zip_path, \"wb\") as f:\n f.write(zip_buffer.getvalue())\n \n try:\n log.reason(\"Importing dashboard\", payload={\"environment\": env.name})\n result = client.import_dashboard(temp_zip_path)\n log.reflect(\"Deployment successful\", payload={\"dashboard_id\": dashboard_id})\n return {\"status\": \"success\", \"message\": f\"Dashboard deployed to {env.name}\", \"details\": result}\n finally:\n if temp_zip_path.exists():\n os.remove(temp_zip_path)\n\n except Exception as e:\n log.explore(\"Deployment failed\", error=str(e))\n raise\n # [/DEF:_handle_deploy:Function]\n" }, { "contract_id": "_get_env", "contract_type": "Function", "file_path": "backend/src/plugins/git_plugin.py", - "start_line": 334, - "end_line": 397, + "start_line": 337, + "end_line": 400, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -59884,7 +60893,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n app_logger.info(f\"[_get_env][Entry] Fetching environment for ID: {env_id}\")\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}\")\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n app_logger.info(f\"[_get_env][Exit] Found environment in DB: {db_env.name}\")\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n app_logger.info(f\"[_get_env][Exit] Found environment {env_id} in ConfigManager list\")\n return env\n \n if not env_id:\n app_logger.info(f\"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}\")\n return envs[0]\n \n app_logger.error(f\"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n" + "body": " # [DEF:_get_env:Function]\n # @PURPOSE: Вспомогательный метод для получения конфигурации окружения.\n # @PARAM: env_id (Optional[str]) - ID окружения.\n # @PRE: env_id is a string or None.\n # @POST: Returns an Environment object from config or DB.\n # @RETURN: Environment - Объект конфигурации окружения.\n def _get_env(self, env_id: Optional[str] = None):\n with belief_scope(\"GitPlugin._get_env\"):\n log.reason(\"Fetching environment\", payload={\"env_id\": env_id})\n \n # Priority 1: ConfigManager (config.json)\n if env_id:\n env = self.config_manager.get_environment(env_id)\n if env:\n log.reflect(\"Found environment by ID in ConfigManager\", payload={\"name\": env.name})\n return env\n \n # Priority 2: Database (DeploymentEnvironment)\n from src.core.database import SessionLocal\n from src.models.git import DeploymentEnvironment\n \n db = SessionLocal()\n try:\n if env_id:\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()\n else:\n # If no ID, try to find active or any environment in DB\n db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()\n if not db_env:\n db_env = db.query(DeploymentEnvironment).first()\n\n if db_env:\n log.reflect(\"Found environment in DB\", payload={\"name\": db_env.name})\n from src.core.config_models import Environment\n # Use token as password for SupersetClient\n return Environment(\n id=db_env.id,\n name=db_env.name,\n url=db_env.superset_url,\n username=\"admin\",\n password=db_env.superset_token,\n verify_ssl=True\n )\n finally:\n db.close()\n \n # Priority 3: ConfigManager Default (if no env_id provided)\n envs = self.config_manager.get_environments()\n if envs:\n if env_id:\n # If env_id was provided but not found in DB or specifically by ID in config,\n # but we have other envs, maybe it's one of them?\n env = next((e for e in envs if e.id == env_id), None)\n if env:\n log.reflect(\"Found environment in ConfigManager list\", payload={\"env_id\": env_id})\n return env\n \n if not env_id:\n log.reflect(\"Using first environment from ConfigManager\", payload={\"name\": envs[0].name})\n return envs[0]\n \n log.explore(\"No environments configured\", payload={\"env_id\": env_id}, error=\"No Superset environments found in config or database\")\n raise ValueError(\"No environments configured. Please add a Superset Environment in Settings.\")\n # [/DEF:_get_env:Function]\n" }, { "contract_id": "backend/src/plugins/llm_analysis/__init__.py", @@ -62026,7 +63035,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/migration.py", "start_line": 1, - "end_line": 390, + "end_line": 393, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -62113,7 +63122,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:MigrationPlugin:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: migration, superset, automation, dashboard, plugin, transformation\n# @PURPOSE: Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments.\n# @LAYER: App\n# @RELATION: IMPLEMENTS -> PluginBase\n# @RELATION: DEPENDS_ON -> SupersetClient\n# @RELATION: DEPENDS_ON -> MigrationEngine\n# @RELATION: DEPENDS_ON -> IdMappingService\n# @RELATION: USES -> TaskContext\n# @PRE: Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.\n# @POST: Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.\n# @SIDE_EFFECT: Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.\n# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]\n# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.\n\nfrom typing import Dict, Any, Optional\nimport re\n\nfrom ..core.plugin_base import PluginBase\nfrom ..core.logger import belief_scope, logger as app_logger\nfrom ..core.superset_client import SupersetClient\nfrom ..core.utils.fileio import create_temp_file\nfrom ..dependencies import get_config_manager\nfrom ..core.migration_engine import MigrationEngine\nfrom ..core.database import SessionLocal\nfrom ..models.mapping import DatabaseMapping, Environment\nfrom ..core.mapping_service import IdMappingService\nfrom ..core.task_manager.context import TaskContext\n\n# [DEF:MigrationPlugin:Class]\n# @PURPOSE: Implementation of the migration plugin workflow and transformation orchestration.\n# @PRE: SupersetClient authenticated, database session active\n# @POST: Returns MigrationResult with success/failure status and artifact list\n# @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip\n# @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {\"db_mappings\": {\"source_uuid_1\": \"target_uuid_2\"}}\n# @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {\"passwords\": {\"PostgreSQL\": \"secret123\"}}\n# @TEST_INVARIANT: strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution]\n# @SIDE_EFFECT: Writes migration artifacts to database, triggers dashboard imports\n# @DATA_CONTRACT: MigrationPlan AST, DryRunResult, RiskAssessment\nclass MigrationPlugin(PluginBase):\n \"\"\"\n A plugin to migrate Superset dashboards between environments.\n \"\"\"\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the unique identifier for the migration plugin.\n # @PRE: None.\n # @POST: Returns stable string \"superset-migration\".\n # @RETURN: str\n def id(self) -> str:\n with belief_scope(\"MigrationPlugin.id\"):\n return \"superset-migration\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the human-readable name of the plugin.\n # @PRE: None.\n # @POST: Returns \"Superset Dashboard Migration\".\n # @RETURN: str\n def name(self) -> str:\n with belief_scope(\"MigrationPlugin.name\"):\n return \"Superset Dashboard Migration\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns the semantic description of the plugin.\n # @PRE: None.\n # @POST: Returns description string.\n # @RETURN: str\n def description(self) -> str:\n with belief_scope(\"MigrationPlugin.description\"):\n return \"Migrates dashboards between Superset environments.\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the semantic version of the migration plugin.\n # @PRE: None.\n # @POST: Returns \"1.0.0\".\n # @RETURN: str\n def version(self) -> str:\n with belief_scope(\"MigrationPlugin.version\"):\n return \"1.0.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend routing anchor for the plugin.\n # @PRE: None.\n # @POST: Returns \"/migration\".\n # @RETURN: str\n def ui_route(self) -> str:\n with belief_scope(\"MigrationPlugin.ui_route\"):\n return \"/migration\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically.\n # @PRE: ConfigManager is accessible and environments are defined.\n # @POST: Returns a JSON Schema dict matching current system environments.\n # @RETURN: Dict[str, Any]\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"MigrationPlugin.get_schema\"):\n app_logger.reason(\"Generating migration UI schema\")\n config_manager = get_config_manager()\n envs = [e.name for e in config_manager.get_environments()]\n \n schema = {\n \"type\": \"object\",\n \"properties\": {\n \"from_env\": {\n \"type\": \"string\",\n \"title\": \"Source Environment\",\n \"description\": \"The environment to migrate from.\",\n \"enum\": envs if envs else [\"dev\", \"prod\"],\n },\n \"to_env\": {\n \"type\": \"string\",\n \"title\": \"Target Environment\",\n \"description\": \"The environment to migrate to.\",\n \"enum\": envs if envs else [\"dev\", \"prod\"],\n },\n \"dashboard_regex\": {\n \"type\": \"string\",\n \"title\": \"Dashboard Regex\",\n \"description\": \"A regular expression to filter dashboards to migrate.\",\n },\n \"replace_db_config\": {\n \"type\": \"boolean\",\n \"title\": \"Replace DB Config\",\n \"description\": \"Whether to replace the database configuration.\",\n \"default\": False,\n },\n \"from_db_id\": {\n \"type\": \"integer\",\n \"title\": \"Source DB ID\",\n \"description\": \"The ID of the source database to replace (if replacing).\",\n },\n \"to_db_id\": {\n \"type\": \"integer\",\n \"title\": \"Target DB ID\",\n \"description\": \"The ID of the target database to replace with (if replacing).\",\n },\n },\n \"required\": [\"from_env\", \"to_env\", \"dashboard_regex\"],\n }\n app_logger.reflect(\"Schema generated successfully\", extra={\"environments_count\": len(envs)})\n return schema\n # [/DEF:get_schema:Function]\n\n # [DEF:execute:Function]\n # @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.\n # @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request.\n # @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.\n # @PRE: Source and target environments must resolve. Matching dashboards must exist.\n # @POST: Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized.\n # @SIDE_EFFECT: Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings).\n # @TEST_CONTRACT: Dict[str, Any] -> Dict[str, Any]\n # @TEST_SCENARIO: successful_dashboard_transfer -> ZIP is downloaded, DB mappings applied via AST, target import succeeds.\n # @TEST_SCENARIO: missing_password_injection -> Target import fails on auth, TaskManager pauses for user input, retries with password successfully.\n # @TEST_SCENARIO: empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards.\n # @TEST_EDGE: missing_env_field -> [ValueError: Could not resolve source or target environment]\n # @TEST_EDGE: invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully]\n # @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]\n async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):\n with belief_scope(\"MigrationPlugin.execute\"):\n app_logger.reason(\"Evaluating migration task parameters\", extra={\"params\": params})\n \n source_env_id = params.get(\"source_env_id\")\n target_env_id = params.get(\"target_env_id\")\n selected_ids = params.get(\"selected_ids\")\n \n from_env_name = params.get(\"from_env\")\n to_env_name = params.get(\"to_env\")\n dashboard_regex = params.get(\"dashboard_regex\")\n replace_db_config = params.get(\"replace_db_config\", False)\n fix_cross_filters = params.get(\"fix_cross_filters\", True)\n \n task_id = params.get(\"_task_id\")\n from ..dependencies import get_task_manager\n tm = get_task_manager()\n\n log = context.logger if context else app_logger\n superset_log = log.with_source(\"superset_api\") if context else log\n migration_log = log.with_source(\"migration\") if context else log\n \n log.info(\"Starting migration task.\")\n\n try:\n config_manager = get_config_manager()\n environments = config_manager.get_environments()\n \n # Resolve environments\n src_env = next((e for e in environments if e.id == source_env_id), None) if source_env_id else next((e for e in environments if e.name == from_env_name), None)\n tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None)\n \n if not src_env or not tgt_env:\n app_logger.explore(\"Environment resolution failed\", extra={\"src\": source_env_id or from_env_name, \"tgt\": target_env_id or to_env_name})\n raise ValueError(f\"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}\")\n\n from_env_name = src_env.name\n to_env_name = tgt_env.name\n\n app_logger.reason(\"Environments resolved successfully\", extra={\"from\": from_env_name, \"to\": to_env_name})\n \n migration_result = {\n \"status\": \"SUCCESS\",\n \"source_environment\": from_env_name,\n \"target_environment\": to_env_name,\n \"selected_dashboards\": 0,\n \"migrated_dashboards\": [],\n \"failed_dashboards\": [],\n \"mapping_count\": 0\n }\n\n from_c = SupersetClient(src_env)\n to_c = SupersetClient(tgt_env)\n \n if not from_c or not to_c:\n raise ValueError(f\"Clients not initialized for environments: {from_env_name}, {to_env_name}\")\n\n _, all_dashboards = from_c.get_dashboards()\n \n # Selection Logic\n if selected_ids:\n dashboards_to_migrate = [d for d in all_dashboards if d[\"id\"] in selected_ids]\n elif dashboard_regex:\n regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE)\n dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get(\"dashboard_title\", \"\"))]\n else:\n app_logger.explore(\"No deterministic selection criteria provided\")\n migration_result[\"status\"] = \"NO_SELECTION\"\n return migration_result\n\n if not dashboards_to_migrate:\n app_logger.explore(\"Zero dashboards match selection criteria\")\n migration_result[\"status\"] = \"NO_MATCHES\"\n return migration_result\n\n migration_result[\"selected_dashboards\"] = len(dashboards_to_migrate)\n\n # Database Mapping Resolution\n db_mapping = params.get(\"db_mappings\", {})\n if not isinstance(db_mapping, dict):\n db_mapping = {}\n \n if replace_db_config:\n app_logger.reason(\"Fetching environment DB mappings from catalog\")\n db = SessionLocal()\n try:\n src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first()\n tgt_env_db = db.query(Environment).filter(Environment.name == to_env_name).first()\n \n if src_env_db and tgt_env_db:\n stored_mappings = db.query(DatabaseMapping).filter(\n DatabaseMapping.source_env_id == src_env_db.id,\n DatabaseMapping.target_env_id == tgt_env_db.id\n ).all()\n stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}\n stored_map_dict.update(db_mapping)\n db_mapping = stored_map_dict\n log.info(f\"Loaded {len(stored_mappings)} database mappings from database.\")\n finally:\n db.close()\n\n migration_result[\"mapping_count\"] = len(db_mapping)\n engine = MigrationEngine()\n\n # Migration Loop\n for dash in dashboards_to_migrate:\n dash_id, dash_slug, title = dash[\"id\"], dash.get(\"slug\"), dash[\"dashboard_title\"]\n app_logger.reason(f\"Starting pipeline for dashboard '{title}'\", extra={\"dash_id\": dash_id})\n \n try:\n exported_content, _ = from_c.export_dashboard(dash_id)\n with create_temp_file(content=exported_content, dry_run=True, suffix=\".zip\") as tmp_zip_path:\n with create_temp_file(suffix=\".zip\", dry_run=True) as tmp_new_zip:\n \n success = engine.transform_zip(\n str(tmp_zip_path), \n str(tmp_new_zip), \n db_mapping, \n strip_databases=False, \n target_env_id=tgt_env.id if tgt_env else None, \n fix_cross_filters=fix_cross_filters\n )\n \n if not success and replace_db_config:\n if task_id:\n app_logger.explore(\"Missing mapping blocks AST transform. Pausing task for user intervention.\", extra={\"task_id\": task_id})\n await tm.wait_for_resolution(task_id)\n \n app_logger.reason(\"Task resumed, re-evaluating mapping states\")\n db = SessionLocal()\n try:\n src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()\n tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first()\n mappings = db.query(DatabaseMapping).filter(\n DatabaseMapping.source_env_id == src_env_rt.id,\n DatabaseMapping.target_env_id == tgt_env_rt.id\n ).all()\n db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings}\n finally:\n db.close()\n \n success = engine.transform_zip(\n str(tmp_zip_path), \n str(tmp_new_zip), \n db_mapping, \n strip_databases=False,\n target_env_id=tgt_env.id if tgt_env else None,\n fix_cross_filters=fix_cross_filters\n )\n\n if success:\n app_logger.reason(\"Pushing transformed ZIP to target Superset\")\n to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)\n migration_result[\"migrated_dashboards\"].append({\"id\": dash_id, \"title\": title})\n app_logger.reflect(\"Import successful\", extra={\"title\": title})\n else:\n app_logger.explore(\"Transformation strictly failed, bypassing ingestion\")\n migration_log.error(f\"Failed to transform ZIP for dashboard {title}\")\n migration_result[\"failed_dashboards\"].append({\n \"id\": dash_id, \"title\": title, \"error\": \"Failed to transform ZIP\"\n })\n \n except Exception as exc:\n error_msg = str(exc)\n if \"Must provide a password for the database\" in error_msg:\n db_name = \"unknown\"\n match = re.search(r\"databases/([^.]+)\\.yaml\", error_msg)\n if match:\n db_name = match.group(1)\n else:\n match_alt = re.search(r\"database '([^']+)'\", error_msg)\n if match_alt:\n db_name = match_alt.group(1)\n\n app_logger.explore(f\"Missing DB password detected during ingestion. Escalating to UI.\", extra={\"db_name\": db_name})\n \n if task_id:\n tm.await_input(task_id, {\n \"type\": \"database_password\",\n \"databases\": [db_name],\n \"error_message\": error_msg\n })\n \n await tm.wait_for_input(task_id)\n task = tm.get_task(task_id)\n passwords = task.params.get(\"passwords\", {})\n \n if passwords:\n app_logger.reason(f\"Retrying import for {title} with injected credentials\")\n to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)\n migration_result[\"migrated_dashboards\"].append({\"id\": dash_id, \"title\": title})\n app_logger.reflect(\"Password injection unblocked import\")\n if \"passwords\" in task.params:\n del task.params[\"passwords\"]\n continue\n\n app_logger.explore(f\"Catastrophic dashboard ingestion failure: {exc}\")\n migration_result[\"failed_dashboards\"].append({\"id\": dash_id, \"title\": title, \"error\": str(exc)})\n\n if migration_result[\"failed_dashboards\"]:\n migration_result[\"status\"] = \"PARTIAL_SUCCESS\"\n\n # Post-Migration ID Mapping Synchronization\n try:\n app_logger.reason(\"Executing incremental ID catalog sync on target\")\n db_session = SessionLocal()\n mapping_service = IdMappingService(db_session)\n mapping_service.sync_environment(tgt_env.id, to_c, incremental=True)\n db_session.close()\n app_logger.reflect(\"Incremental catalog sync closed out cleanly\")\n except Exception as sync_exc:\n app_logger.explore(f\"ID Mapping sync failed, mapping state might be degraded: {sync_exc}\")\n\n app_logger.reflect(\"Migration cycle fully resolved\", extra={\"result\": migration_result})\n return migration_result\n\n except Exception as e:\n app_logger.explore(f\"Fatal plugin failure: {e}\", exc_info=True)\n raise e\n # [/DEF:execute:Function]\n# [/DEF:MigrationPlugin:Class]\n# [/DEF:MigrationPlugin:Module]\n" + "body": "# [DEF:MigrationPlugin:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: migration, superset, automation, dashboard, plugin, transformation\n# @PURPOSE: Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments.\n# @LAYER: App\n# @RELATION: IMPLEMENTS -> PluginBase\n# @RELATION: DEPENDS_ON -> SupersetClient\n# @RELATION: DEPENDS_ON -> MigrationEngine\n# @RELATION: DEPENDS_ON -> IdMappingService\n# @RELATION: USES -> TaskContext\n# @PRE: Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context.\n# @POST: Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees.\n# @SIDE_EFFECT: Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows.\n# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set]\n# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution.\n\nfrom typing import Dict, Any, Optional\nimport re\n\nfrom ..core.plugin_base import PluginBase\nfrom ..core.logger import belief_scope\nfrom ..core.cot_logger import MarkerLogger\nfrom ..core.superset_client import SupersetClient\n\nlog = MarkerLogger(\"MigrationPlugin\")\nfrom ..core.utils.fileio import create_temp_file\nfrom ..dependencies import get_config_manager\nfrom ..core.migration_engine import MigrationEngine\nfrom ..core.database import SessionLocal\nfrom ..models.mapping import DatabaseMapping, Environment\nfrom ..core.mapping_service import IdMappingService\nfrom ..core.task_manager.context import TaskContext\n\n# [DEF:MigrationPlugin:Class]\n# @PURPOSE: Implementation of the migration plugin workflow and transformation orchestration.\n# @PRE: SupersetClient authenticated, database session active\n# @POST: Returns MigrationResult with success/failure status and artifact list\n# @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip\n# @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {\"db_mappings\": {\"source_uuid_1\": \"target_uuid_2\"}}\n# @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {\"passwords\": {\"PostgreSQL\": \"secret123\"}}\n# @TEST_INVARIANT: strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution]\n# @SIDE_EFFECT: Writes migration artifacts to database, triggers dashboard imports\n# @DATA_CONTRACT: MigrationPlan AST, DryRunResult, RiskAssessment\nclass MigrationPlugin(PluginBase):\n \"\"\"\n A plugin to migrate Superset dashboards between environments.\n \"\"\"\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the unique identifier for the migration plugin.\n # @PRE: None.\n # @POST: Returns stable string \"superset-migration\".\n # @RETURN: str\n def id(self) -> str:\n with belief_scope(\"MigrationPlugin.id\"):\n return \"superset-migration\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the human-readable name of the plugin.\n # @PRE: None.\n # @POST: Returns \"Superset Dashboard Migration\".\n # @RETURN: str\n def name(self) -> str:\n with belief_scope(\"MigrationPlugin.name\"):\n return \"Superset Dashboard Migration\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns the semantic description of the plugin.\n # @PRE: None.\n # @POST: Returns description string.\n # @RETURN: str\n def description(self) -> str:\n with belief_scope(\"MigrationPlugin.description\"):\n return \"Migrates dashboards between Superset environments.\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the semantic version of the migration plugin.\n # @PRE: None.\n # @POST: Returns \"1.0.0\".\n # @RETURN: str\n def version(self) -> str:\n with belief_scope(\"MigrationPlugin.version\"):\n return \"1.0.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend routing anchor for the plugin.\n # @PRE: None.\n # @POST: Returns \"/migration\".\n # @RETURN: str\n def ui_route(self) -> str:\n with belief_scope(\"MigrationPlugin.ui_route\"):\n return \"/migration\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically.\n # @PRE: ConfigManager is accessible and environments are defined.\n # @POST: Returns a JSON Schema dict matching current system environments.\n # @RETURN: Dict[str, Any]\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"MigrationPlugin.get_schema\"):\n log.reason(\"Generating migration UI schema\")\n config_manager = get_config_manager()\n envs = [e.name for e in config_manager.get_environments()]\n \n schema = {\n \"type\": \"object\",\n \"properties\": {\n \"from_env\": {\n \"type\": \"string\",\n \"title\": \"Source Environment\",\n \"description\": \"The environment to migrate from.\",\n \"enum\": envs if envs else [\"dev\", \"prod\"],\n },\n \"to_env\": {\n \"type\": \"string\",\n \"title\": \"Target Environment\",\n \"description\": \"The environment to migrate to.\",\n \"enum\": envs if envs else [\"dev\", \"prod\"],\n },\n \"dashboard_regex\": {\n \"type\": \"string\",\n \"title\": \"Dashboard Regex\",\n \"description\": \"A regular expression to filter dashboards to migrate.\",\n },\n \"replace_db_config\": {\n \"type\": \"boolean\",\n \"title\": \"Replace DB Config\",\n \"description\": \"Whether to replace the database configuration.\",\n \"default\": False,\n },\n \"from_db_id\": {\n \"type\": \"integer\",\n \"title\": \"Source DB ID\",\n \"description\": \"The ID of the source database to replace (if replacing).\",\n },\n \"to_db_id\": {\n \"type\": \"integer\",\n \"title\": \"Target DB ID\",\n \"description\": \"The ID of the target database to replace with (if replacing).\",\n },\n },\n \"required\": [\"from_env\", \"to_env\", \"dashboard_regex\"],\n }\n log.reflect(\"Schema generated successfully\", payload={\"environments_count\": len(envs)})\n return schema\n # [/DEF:get_schema:Function]\n\n # [DEF:execute:Function]\n # @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion.\n # @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request.\n # @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing.\n # @PRE: Source and target environments must resolve. Matching dashboards must exist.\n # @POST: Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized.\n # @SIDE_EFFECT: Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings).\n # @TEST_CONTRACT: Dict[str, Any] -> Dict[str, Any]\n # @TEST_SCENARIO: successful_dashboard_transfer -> ZIP is downloaded, DB mappings applied via AST, target import succeeds.\n # @TEST_SCENARIO: missing_password_injection -> Target import fails on auth, TaskManager pauses for user input, retries with password successfully.\n # @TEST_SCENARIO: empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards.\n # @TEST_EDGE: missing_env_field -> [ValueError: Could not resolve source or target environment]\n # @TEST_EDGE: invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully]\n # @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]\n async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):\n with belief_scope(\"MigrationPlugin.execute\"):\n log.reason(\"Evaluating migration task parameters\", payload={\"params\": params})\n \n source_env_id = params.get(\"source_env_id\")\n target_env_id = params.get(\"target_env_id\")\n selected_ids = params.get(\"selected_ids\")\n \n from_env_name = params.get(\"from_env\")\n to_env_name = params.get(\"to_env\")\n dashboard_regex = params.get(\"dashboard_regex\")\n replace_db_config = params.get(\"replace_db_config\", False)\n fix_cross_filters = params.get(\"fix_cross_filters\", True)\n \n task_id = params.get(\"_task_id\")\n from ..dependencies import get_task_manager\n tm = get_task_manager()\n\n exec_log = context.logger if context else log\n superset_log = exec_log.with_source(\"superset_api\") if context else exec_log\n migration_log = exec_log.with_source(\"migration\") if context else exec_log\n \n exec_log.info(\"Starting migration task.\")\n\n try:\n config_manager = get_config_manager()\n environments = config_manager.get_environments()\n \n # Resolve environments\n src_env = next((e for e in environments if e.id == source_env_id), None) if source_env_id else next((e for e in environments if e.name == from_env_name), None)\n tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None)\n \n if not src_env or not tgt_env:\n log.explore(\"Environment resolution failed\", error=\"Could not resolve source or target environment\", payload={\"src\": source_env_id or from_env_name, \"tgt\": target_env_id or to_env_name})\n raise ValueError(f\"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}\")\n\n from_env_name = src_env.name\n to_env_name = tgt_env.name\n\n log.reason(\"Environments resolved successfully\", payload={\"from\": from_env_name, \"to\": to_env_name})\n \n migration_result = {\n \"status\": \"SUCCESS\",\n \"source_environment\": from_env_name,\n \"target_environment\": to_env_name,\n \"selected_dashboards\": 0,\n \"migrated_dashboards\": [],\n \"failed_dashboards\": [],\n \"mapping_count\": 0\n }\n\n from_c = SupersetClient(src_env)\n to_c = SupersetClient(tgt_env)\n \n if not from_c or not to_c:\n raise ValueError(f\"Clients not initialized for environments: {from_env_name}, {to_env_name}\")\n\n _, all_dashboards = from_c.get_dashboards()\n \n # Selection Logic\n if selected_ids:\n dashboards_to_migrate = [d for d in all_dashboards if d[\"id\"] in selected_ids]\n elif dashboard_regex:\n regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE)\n dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get(\"dashboard_title\", \"\"))]\n else:\n log.explore(\"No deterministic selection criteria provided\", error=\"No dashboard selection criteria (selected_ids or dashboard_regex)\")\n migration_result[\"status\"] = \"NO_SELECTION\"\n return migration_result\n\n if not dashboards_to_migrate:\n log.explore(\"Zero dashboards match selection criteria\", error=\"No dashboards matched the given selection criteria\")\n migration_result[\"status\"] = \"NO_MATCHES\"\n return migration_result\n\n migration_result[\"selected_dashboards\"] = len(dashboards_to_migrate)\n\n # Database Mapping Resolution\n db_mapping = params.get(\"db_mappings\", {})\n if not isinstance(db_mapping, dict):\n db_mapping = {}\n \n if replace_db_config:\n log.reason(\"Fetching environment DB mappings from catalog\")\n db = SessionLocal()\n try:\n src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first()\n tgt_env_db = db.query(Environment).filter(Environment.name == to_env_name).first()\n \n if src_env_db and tgt_env_db:\n stored_mappings = db.query(DatabaseMapping).filter(\n DatabaseMapping.source_env_id == src_env_db.id,\n DatabaseMapping.target_env_id == tgt_env_db.id\n ).all()\n stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}\n stored_map_dict.update(db_mapping)\n db_mapping = stored_map_dict\n exec_log.info(f\"Loaded {len(stored_mappings)} database mappings from database.\")\n finally:\n db.close()\n\n migration_result[\"mapping_count\"] = len(db_mapping)\n engine = MigrationEngine()\n\n # Migration Loop\n for dash in dashboards_to_migrate:\n dash_id, dash_slug, title = dash[\"id\"], dash.get(\"slug\"), dash[\"dashboard_title\"]\n log.reason(f\"Starting pipeline for dashboard '{title}'\", payload={\"dash_id\": dash_id})\n \n try:\n exported_content, _ = from_c.export_dashboard(dash_id)\n with create_temp_file(content=exported_content, dry_run=True, suffix=\".zip\") as tmp_zip_path:\n with create_temp_file(suffix=\".zip\", dry_run=True) as tmp_new_zip:\n \n success = engine.transform_zip(\n str(tmp_zip_path), \n str(tmp_new_zip), \n db_mapping, \n strip_databases=False, \n target_env_id=tgt_env.id if tgt_env else None, \n fix_cross_filters=fix_cross_filters\n )\n \n if not success and replace_db_config:\n if task_id:\n log.explore(\"Missing mapping blocks AST transform. Pausing task for user intervention.\", error=\"AST transform blocked by missing mappings\", payload={\"task_id\": task_id})\n await tm.wait_for_resolution(task_id)\n \n log.reason(\"Task resumed, re-evaluating mapping states\")\n db = SessionLocal()\n try:\n src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()\n tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first()\n mappings = db.query(DatabaseMapping).filter(\n DatabaseMapping.source_env_id == src_env_rt.id,\n DatabaseMapping.target_env_id == tgt_env_rt.id\n ).all()\n db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings}\n finally:\n db.close()\n \n success = engine.transform_zip(\n str(tmp_zip_path), \n str(tmp_new_zip), \n db_mapping, \n strip_databases=False,\n target_env_id=tgt_env.id if tgt_env else None,\n fix_cross_filters=fix_cross_filters\n )\n\n if success:\n log.reason(\"Pushing transformed ZIP to target Superset\")\n to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)\n migration_result[\"migrated_dashboards\"].append({\"id\": dash_id, \"title\": title})\n log.reflect(\"Import successful\", payload={\"title\": title})\n else:\n log.explore(\"Transformation strictly failed, bypassing ingestion\", error=\"Dashboard ZIP transformation failed\")\n migration_log.error(f\"Failed to transform ZIP for dashboard {title}\")\n migration_result[\"failed_dashboards\"].append({\n \"id\": dash_id, \"title\": title, \"error\": \"Failed to transform ZIP\"\n })\n \n except Exception as exc:\n error_msg = str(exc)\n if \"Must provide a password for the database\" in error_msg:\n db_name = \"unknown\"\n match = re.search(r\"databases/([^.]+)\\.yaml\", error_msg)\n if match:\n db_name = match.group(1)\n else:\n match_alt = re.search(r\"database '([^']+)'\", error_msg)\n if match_alt:\n db_name = match_alt.group(1)\n\n log.explore(\"Missing DB password detected during ingestion. Escalating to UI.\", error=\"Missing DB password for import\", payload={\"db_name\": db_name})\n \n if task_id:\n tm.await_input(task_id, {\n \"type\": \"database_password\",\n \"databases\": [db_name],\n \"error_message\": error_msg\n })\n \n await tm.wait_for_input(task_id)\n task = tm.get_task(task_id)\n passwords = task.params.get(\"passwords\", {})\n \n if passwords:\n log.reason(f\"Retrying import for {title} with injected credentials\")\n to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)\n migration_result[\"migrated_dashboards\"].append({\"id\": dash_id, \"title\": title})\n log.reflect(\"Password injection unblocked import\")\n if \"passwords\" in task.params:\n del task.params[\"passwords\"]\n continue\n\n log.explore(\"Catastrophic dashboard ingestion failure\", error=f\"Dashboard ingestion failed: {exc}\")\n migration_result[\"failed_dashboards\"].append({\"id\": dash_id, \"title\": title, \"error\": str(exc)})\n\n if migration_result[\"failed_dashboards\"]:\n migration_result[\"status\"] = \"PARTIAL_SUCCESS\"\n\n # Post-Migration ID Mapping Synchronization\n try:\n log.reason(\"Executing incremental ID catalog sync on target\")\n db_session = SessionLocal()\n mapping_service = IdMappingService(db_session)\n mapping_service.sync_environment(tgt_env.id, to_c, incremental=True)\n db_session.close()\n log.reflect(\"Incremental catalog sync closed out cleanly\")\n except Exception as sync_exc:\n log.explore(\"ID Mapping sync failed, mapping state might be degraded\", error=f\"ID Mapping sync error: {sync_exc}\")\n\n log.reflect(\"Migration cycle fully resolved\", payload={\"result\": migration_result})\n return migration_result\n\n except Exception as e:\n log.explore(\"Fatal plugin failure\", error=f\"Plugin execution failed: {e}\", exc_info=True)\n raise e\n # [/DEF:execute:Function]\n# [/DEF:MigrationPlugin:Class]\n# [/DEF:MigrationPlugin:Module]\n" }, { "contract_id": "SearchPluginModule", @@ -62265,7 +63274,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/storage/plugin.py", "start_line": 1, - "end_line": 391, + "end_line": 387, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -62351,14 +63360,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:StoragePlugin:Module]\n#\n# @SEMANTICS: storage, files, filesystem, plugin\n# @PURPOSE: Provides core filesystem operations for managing backups and repositories.\n# @LAYER: App\n# @RELATION: IMPLEMENTS -> PluginBase\n# @RELATION: DEPENDS_ON -> backend.src.models.storage\n# @RELATION: USES -> TaskContext\n#\n# @INVARIANT: All file operations must be restricted to the configured storage root.\n\n# [SECTION: IMPORTS]\nimport os\nimport shutil\nfrom pathlib import Path\nfrom datetime import datetime\nfrom typing import Dict, Any, List, Optional\nfrom fastapi import UploadFile\n\nfrom ...core.plugin_base import PluginBase\nfrom ...core.logger import belief_scope, logger\nfrom ...models.storage import StoredFile, FileCategory\nfrom ...dependencies import get_config_manager\nfrom ...core.task_manager.context import TaskContext\n# [/SECTION]\n\n# [DEF:StoragePlugin:Class]\n# @PURPOSE: Implementation of the storage management plugin.\nclass StoragePlugin(PluginBase):\n \"\"\"\n Plugin for managing local file storage for backups and repositories.\n \"\"\"\n\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the StoragePlugin and ensures required directories exist.\n # @PRE: Configuration manager must be accessible.\n # @POST: Storage root and category directories are created on disk.\n def __init__(self):\n with belief_scope(\"StoragePlugin:init\"):\n self.ensure_directories()\n # [/DEF:__init__:Function]\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the unique identifier for the storage plugin.\n # @PRE: None.\n # @POST: Returns the plugin ID string.\n # @RETURN: str - \"storage-manager\"\n def id(self) -> str:\n with belief_scope(\"StoragePlugin:id\"):\n return \"storage-manager\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the human-readable name of the storage plugin.\n # @PRE: None.\n # @POST: Returns the plugin name string.\n # @RETURN: str - \"Storage Manager\"\n def name(self) -> str:\n with belief_scope(\"StoragePlugin:name\"):\n return \"Storage Manager\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns a description of the storage plugin.\n # @PRE: None.\n # @POST: Returns the plugin description string.\n # @RETURN: str - Plugin description.\n def description(self) -> str:\n with belief_scope(\"StoragePlugin:description\"):\n return \"Manages local file storage for backups and repositories.\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the version of the storage plugin.\n # @PRE: None.\n # @POST: Returns the version string.\n # @RETURN: str - \"1.0.0\"\n def version(self) -> str:\n with belief_scope(\"StoragePlugin:version\"):\n return \"1.0.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend route for the storage plugin.\n # @RETURN: str - \"/tools/storage\"\n def ui_route(self) -> str:\n with belief_scope(\"StoragePlugin:ui_route\"):\n return \"/tools/storage\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Returns the JSON schema for storage plugin parameters.\n # @PRE: None.\n # @POST: Returns a dictionary representing the JSON schema.\n # @RETURN: Dict[str, Any] - JSON schema.\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"StoragePlugin:get_schema\"):\n return {\n \"type\": \"object\",\n \"properties\": {\n \"category\": {\n \"type\": \"string\",\n \"enum\": [c.value for c in FileCategory],\n \"title\": \"Category\"\n }\n },\n \"required\": [\"category\"]\n }\n # [/DEF:get_schema:Function]\n\n # [DEF:execute:Function]\n # @PURPOSE: Executes storage-related tasks with TaskContext support.\n # @PARAM: params (Dict[str, Any]) - Storage parameters.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @PRE: params must match the plugin schema.\n # @POST: Task is executed and logged.\n async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):\n with belief_scope(\"StoragePlugin:execute\"):\n # Use TaskContext logger if available, otherwise fall back to app logger\n log = context.logger if context else logger\n \n # Create sub-loggers for different components\n storage_log = log.with_source(\"storage\") if context else log\n log.with_source(\"filesystem\") if context else log\n \n storage_log.info(f\"Executing with params: {params}\")\n # [/DEF:execute:Function]\n\n # [DEF:get_storage_root:Function]\n # @PURPOSE: Resolves the absolute path to the storage root.\n # @PRE: Settings must define a storage root path.\n # @POST: Returns a Path object representing the storage root.\n def get_storage_root(self) -> Path:\n with belief_scope(\"StoragePlugin:get_storage_root\"):\n config_manager = get_config_manager()\n global_settings = config_manager.get_config().settings\n \n # Use storage.root_path as the source of truth for storage UI\n root = Path(global_settings.storage.root_path)\n \n if not root.is_absolute():\n # Resolve relative to the backend directory\n # Path(__file__) is backend/src/plugins/storage/plugin.py\n # parents[3] is the project root (ss-tools)\n # We need to ensure it's relative to where backend/ is\n project_root = Path(__file__).parents[3]\n root = (project_root / root).resolve()\n return root\n # [/DEF:get_storage_root:Function]\n\n # [DEF:resolve_path:Function]\n # @PURPOSE: Resolves a dynamic path pattern using provided variables.\n # @PARAM: pattern (str) - The path pattern to resolve.\n # @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.\n # @PRE: pattern must be a valid format string.\n # @POST: Returns the resolved path string.\n # @RETURN: str - The resolved path.\n def resolve_path(self, pattern: str, variables: Dict[str, str]) -> str:\n with belief_scope(\"StoragePlugin:resolve_path\"):\n # Add common variables\n vars_with_defaults = {\n \"timestamp\": datetime.now().strftime(\"%Y%m%dT%H%M%S\"),\n **variables\n }\n try:\n resolved = pattern.format(**vars_with_defaults)\n # Clean up any double slashes or leading/trailing slashes for relative path\n return os.path.normpath(resolved).strip(\"/\")\n except KeyError as e:\n logger.warning(f\"[StoragePlugin][Coherence:Failed] Missing variable for path resolution: {e}\")\n # Fallback to literal pattern if formatting fails partially (or handle as needed)\n return pattern.replace(\"{\", \"\").replace(\"}\", \"\")\n # [/DEF:resolve_path:Function]\n\n # [DEF:ensure_directories:Function]\n # @PURPOSE: Creates the storage root and category subdirectories if they don't exist.\n # @PRE: Storage root must be resolvable.\n # @POST: Directories are created on the filesystem.\n # @SIDE_EFFECT: Creates directories on the filesystem.\n def ensure_directories(self):\n with belief_scope(\"StoragePlugin:ensure_directories\"):\n root = self.get_storage_root()\n for category in FileCategory:\n # Use singular name for consistency with BackupPlugin and GitService\n path = root / category.value\n path.mkdir(parents=True, exist_ok=True)\n logger.debug(f\"[StoragePlugin][Action] Ensured directory: {path}\")\n # [/DEF:ensure_directories:Function]\n\n # [DEF:validate_path:Function]\n # @PURPOSE: Prevents path traversal attacks by ensuring the path is within the storage root.\n # @PRE: path must be a Path object.\n # @POST: Returns the resolved absolute path if valid, otherwise raises ValueError.\n def validate_path(self, path: Path) -> Path:\n with belief_scope(\"StoragePlugin:validate_path\"):\n root = self.get_storage_root().resolve()\n resolved = path.resolve()\n try:\n resolved.relative_to(root)\n except ValueError:\n logger.error(f\"[StoragePlugin][Coherence:Failed] Path traversal detected: {resolved} is not under {root}\")\n raise ValueError(\"Access denied: Path is outside of storage root.\")\n return resolved\n # [/DEF:validate_path:Function]\n\n # [DEF:list_files:Function]\n # @PURPOSE: Lists all files and directories in a specific category and subpath.\n # @PARAM: category (Optional[FileCategory]) - The category to list.\n # @PARAM: subpath (Optional[str]) - Nested path within the category.\n # @PARAM: recursive (bool) - Whether to scan nested subdirectories recursively.\n # @PRE: Storage root must exist.\n # @POST: Returns a list of StoredFile objects.\n # @RETURN: List[StoredFile] - List of file and directory metadata objects.\n def list_files(\n self,\n category: Optional[FileCategory] = None,\n subpath: Optional[str] = None,\n recursive: bool = False,\n ) -> List[StoredFile]:\n with belief_scope(\"StoragePlugin:list_files\"):\n root = self.get_storage_root()\n logger.info(\n f\"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}\"\n )\n files = []\n\n # Root view contract: show category directories only.\n if category is None and not subpath:\n for cat in FileCategory:\n base_dir = root / cat.value\n if not base_dir.exists():\n continue\n stat = base_dir.stat()\n files.append(\n StoredFile(\n name=cat.value,\n path=cat.value,\n size=0,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=cat,\n mime_type=\"directory\",\n )\n )\n return sorted(files, key=lambda x: x.name)\n \n categories = [category] if category else list(FileCategory)\n \n for cat in categories:\n # Scan the category subfolder + optional subpath\n base_dir = root / cat.value\n if subpath:\n target_dir = self.validate_path(base_dir / subpath)\n else:\n target_dir = base_dir\n\n if not target_dir.exists():\n continue\n \n logger.debug(f\"[StoragePlugin][Action] Scanning directory: {target_dir}\")\n\n if recursive:\n for current_root, dirs, filenames in os.walk(target_dir):\n dirs[:] = [d for d in dirs if \"Logs\" not in d]\n for filename in filenames:\n file_path = Path(current_root) / filename\n if \"Logs\" in str(file_path):\n continue\n stat = file_path.stat()\n files.append(\n StoredFile(\n name=filename,\n path=str(file_path.relative_to(root)),\n size=stat.st_size,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=cat,\n mime_type=None,\n )\n )\n continue\n\n # Use os.scandir for better performance and to distinguish files vs dirs\n with os.scandir(target_dir) as it:\n for entry in it:\n # Skip logs\n if \"Logs\" in entry.path:\n continue\n\n stat = entry.stat()\n is_dir = entry.is_dir()\n\n files.append(StoredFile(\n name=entry.name,\n path=str(Path(entry.path).relative_to(root)),\n size=stat.st_size if not is_dir else 0,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=cat,\n mime_type=\"directory\" if is_dir else None\n ))\n \n # Sort: directories first, then by name\n return sorted(files, key=lambda x: (x.mime_type != \"directory\", x.name))\n # [/DEF:list_files:Function]\n\n # [DEF:save_file:Function]\n # @PURPOSE: Saves an uploaded file to the specified category and optional subpath.\n # @PARAM: file (UploadFile) - The uploaded file.\n # @PARAM: category (FileCategory) - The target category.\n # @PARAM: subpath (Optional[str]) - The target subpath.\n # @PRE: file must be a valid UploadFile; category must be valid.\n # @POST: File is written to disk and metadata is returned.\n # @RETURN: StoredFile - Metadata of the saved file.\n # @SIDE_EFFECT: Writes file to disk.\n async def save_file(self, file: UploadFile, category: FileCategory, subpath: Optional[str] = None) -> StoredFile:\n with belief_scope(\"StoragePlugin:save_file\"):\n root = self.get_storage_root()\n dest_dir = root / category.value\n if subpath:\n dest_dir = dest_dir / subpath\n \n dest_dir.mkdir(parents=True, exist_ok=True)\n \n dest_path = self.validate_path(dest_dir / file.filename)\n \n with dest_path.open(\"wb\") as buffer:\n shutil.copyfileobj(file.file, buffer)\n \n stat = dest_path.stat()\n return StoredFile(\n name=dest_path.name,\n path=str(dest_path.relative_to(root)),\n size=stat.st_size,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=category,\n mime_type=file.content_type\n )\n # [/DEF:save_file:Function]\n\n # [DEF:delete_file:Function]\n # @PURPOSE: Deletes a file or directory from the specified category and path.\n # @PARAM: category (FileCategory) - The category.\n # @PARAM: path (str) - The relative path of the file or directory.\n # @PRE: path must belong to the specified category and exist on disk.\n # @POST: The file or directory is removed from disk.\n # @SIDE_EFFECT: Removes item from disk.\n def delete_file(self, category: FileCategory, path: str):\n with belief_scope(\"StoragePlugin:delete_file\"):\n root = self.get_storage_root()\n # path is relative to root, but we ensure it starts with category\n full_path = self.validate_path(root / path)\n \n if not str(Path(path)).startswith(category.value):\n raise ValueError(f\"Path {path} does not belong to category {category}\")\n\n if full_path.exists():\n if full_path.is_dir():\n shutil.rmtree(full_path)\n else:\n full_path.unlink()\n logger.info(f\"[StoragePlugin][Action] Deleted: {full_path}\")\n else:\n raise FileNotFoundError(f\"Item {path} not found\")\n # [/DEF:delete_file:Function]\n\n # [DEF:get_file_path:Function]\n # @PURPOSE: Returns the absolute path of a file for download.\n # @PARAM: category (FileCategory) - The category.\n # @PARAM: path (str) - The relative path of the file.\n # @PRE: path must belong to the specified category and be a file.\n # @POST: Returns the absolute Path to the file.\n # @RETURN: Path - Absolute path to the file.\n def get_file_path(self, category: FileCategory, path: str) -> Path:\n with belief_scope(\"StoragePlugin:get_file_path\"):\n root = self.get_storage_root()\n file_path = self.validate_path(root / path)\n \n if not str(Path(path)).startswith(category.value):\n raise ValueError(f\"Path {path} does not belong to category {category}\")\n\n if not file_path.exists() or file_path.is_dir():\n raise FileNotFoundError(f\"File {path} not found\")\n \n return file_path\n # [/DEF:get_file_path:Function]\n\n# [/DEF:StoragePlugin:Class]\n# [/DEF:StoragePlugin:Module]\n" + "body": "# [DEF:StoragePlugin:Module]\n#\n# @SEMANTICS: storage, files, filesystem, plugin\n# @PURPOSE: Provides core filesystem operations for managing backups and repositories.\n# @LAYER: App\n# @RELATION: IMPLEMENTS -> PluginBase\n# @RELATION: DEPENDS_ON -> backend.src.models.storage\n# @RELATION: USES -> TaskContext\n#\n# @INVARIANT: All file operations must be restricted to the configured storage root.\n\n# [SECTION: IMPORTS]\nimport os\nimport shutil\nfrom pathlib import Path\nfrom datetime import datetime\nfrom typing import Dict, Any, List, Optional\nfrom fastapi import UploadFile\n\nfrom ...core.plugin_base import PluginBase\nfrom ...core.logger import belief_scope\nfrom ...core.cot_logger import MarkerLogger\nfrom ...models.storage import StoredFile, FileCategory\n\nlog = MarkerLogger(\"StoragePlugin\")\nfrom ...dependencies import get_config_manager\nfrom ...core.task_manager.context import TaskContext\n# [/SECTION]\n\n# [DEF:StoragePlugin:Class]\n# @PURPOSE: Implementation of the storage management plugin.\nclass StoragePlugin(PluginBase):\n \"\"\"\n Plugin for managing local file storage for backups and repositories.\n \"\"\"\n\n # [DEF:__init__:Function]\n # @PURPOSE: Initializes the StoragePlugin and ensures required directories exist.\n # @PRE: Configuration manager must be accessible.\n # @POST: Storage root and category directories are created on disk.\n def __init__(self):\n with belief_scope(\"StoragePlugin:init\"):\n self.ensure_directories()\n # [/DEF:__init__:Function]\n\n @property\n # [DEF:id:Function]\n # @PURPOSE: Returns the unique identifier for the storage plugin.\n # @PRE: None.\n # @POST: Returns the plugin ID string.\n # @RETURN: str - \"storage-manager\"\n def id(self) -> str:\n with belief_scope(\"StoragePlugin:id\"):\n return \"storage-manager\"\n # [/DEF:id:Function]\n\n @property\n # [DEF:name:Function]\n # @PURPOSE: Returns the human-readable name of the storage plugin.\n # @PRE: None.\n # @POST: Returns the plugin name string.\n # @RETURN: str - \"Storage Manager\"\n def name(self) -> str:\n with belief_scope(\"StoragePlugin:name\"):\n return \"Storage Manager\"\n # [/DEF:name:Function]\n\n @property\n # [DEF:description:Function]\n # @PURPOSE: Returns a description of the storage plugin.\n # @PRE: None.\n # @POST: Returns the plugin description string.\n # @RETURN: str - Plugin description.\n def description(self) -> str:\n with belief_scope(\"StoragePlugin:description\"):\n return \"Manages local file storage for backups and repositories.\"\n # [/DEF:description:Function]\n\n @property\n # [DEF:version:Function]\n # @PURPOSE: Returns the version of the storage plugin.\n # @PRE: None.\n # @POST: Returns the version string.\n # @RETURN: str - \"1.0.0\"\n def version(self) -> str:\n with belief_scope(\"StoragePlugin:version\"):\n return \"1.0.0\"\n # [/DEF:version:Function]\n\n @property\n # [DEF:ui_route:Function]\n # @PURPOSE: Returns the frontend route for the storage plugin.\n # @RETURN: str - \"/tools/storage\"\n def ui_route(self) -> str:\n with belief_scope(\"StoragePlugin:ui_route\"):\n return \"/tools/storage\"\n # [/DEF:ui_route:Function]\n\n # [DEF:get_schema:Function]\n # @PURPOSE: Returns the JSON schema for storage plugin parameters.\n # @PRE: None.\n # @POST: Returns a dictionary representing the JSON schema.\n # @RETURN: Dict[str, Any] - JSON schema.\n def get_schema(self) -> Dict[str, Any]:\n with belief_scope(\"StoragePlugin:get_schema\"):\n return {\n \"type\": \"object\",\n \"properties\": {\n \"category\": {\n \"type\": \"string\",\n \"enum\": [c.value for c in FileCategory],\n \"title\": \"Category\"\n }\n },\n \"required\": [\"category\"]\n }\n # [/DEF:get_schema:Function]\n\n # [DEF:execute:Function]\n # @PURPOSE: Executes storage-related tasks with TaskContext support.\n # @PARAM: params (Dict[str, Any]) - Storage parameters.\n # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.\n # @PRE: params must match the plugin schema.\n # @POST: Task is executed and logged.\n async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):\n with belief_scope(\"StoragePlugin:execute\"):\n log.reason(f\"Executing with params: {params}\")\n # [/DEF:execute:Function]\n\n # [DEF:get_storage_root:Function]\n # @PURPOSE: Resolves the absolute path to the storage root.\n # @PRE: Settings must define a storage root path.\n # @POST: Returns a Path object representing the storage root.\n def get_storage_root(self) -> Path:\n with belief_scope(\"StoragePlugin:get_storage_root\"):\n config_manager = get_config_manager()\n global_settings = config_manager.get_config().settings\n \n # Use storage.root_path as the source of truth for storage UI\n root = Path(global_settings.storage.root_path)\n \n if not root.is_absolute():\n # Resolve relative to the backend directory\n # Path(__file__) is backend/src/plugins/storage/plugin.py\n # parents[3] is the project root (ss-tools)\n # We need to ensure it's relative to where backend/ is\n project_root = Path(__file__).parents[3]\n root = (project_root / root).resolve()\n return root\n # [/DEF:get_storage_root:Function]\n\n # [DEF:resolve_path:Function]\n # @PURPOSE: Resolves a dynamic path pattern using provided variables.\n # @PARAM: pattern (str) - The path pattern to resolve.\n # @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.\n # @PRE: pattern must be a valid format string.\n # @POST: Returns the resolved path string.\n # @RETURN: str - The resolved path.\n def resolve_path(self, pattern: str, variables: Dict[str, str]) -> str:\n with belief_scope(\"StoragePlugin:resolve_path\"):\n # Add common variables\n vars_with_defaults = {\n \"timestamp\": datetime.now().strftime(\"%Y%m%dT%H%M%S\"),\n **variables\n }\n try:\n resolved = pattern.format(**vars_with_defaults)\n # Clean up any double slashes or leading/trailing slashes for relative path\n return os.path.normpath(resolved).strip(\"/\")\n except KeyError as e:\n log.explore(\"Missing variable for path resolution\", error=str(e))\n # Fallback to literal pattern if formatting fails partially (or handle as needed)\n return pattern.replace(\"{\", \"\").replace(\"}\", \"\")\n # [/DEF:resolve_path:Function]\n\n # [DEF:ensure_directories:Function]\n # @PURPOSE: Creates the storage root and category subdirectories if they don't exist.\n # @PRE: Storage root must be resolvable.\n # @POST: Directories are created on the filesystem.\n # @SIDE_EFFECT: Creates directories on the filesystem.\n def ensure_directories(self):\n with belief_scope(\"StoragePlugin:ensure_directories\"):\n root = self.get_storage_root()\n for category in FileCategory:\n # Use singular name for consistency with BackupPlugin and GitService\n path = root / category.value\n path.mkdir(parents=True, exist_ok=True)\n log.reason(f\"Ensured directory: {path}\")\n # [/DEF:ensure_directories:Function]\n\n # [DEF:validate_path:Function]\n # @PURPOSE: Prevents path traversal attacks by ensuring the path is within the storage root.\n # @PRE: path must be a Path object.\n # @POST: Returns the resolved absolute path if valid, otherwise raises ValueError.\n def validate_path(self, path: Path) -> Path:\n with belief_scope(\"StoragePlugin:validate_path\"):\n root = self.get_storage_root().resolve()\n resolved = path.resolve()\n try:\n resolved.relative_to(root)\n except ValueError:\n log.explore(f\"Path traversal detected: {resolved} is not under {root}\", error=\"Path outside storage root\")\n raise ValueError(\"Access denied: Path is outside of storage root.\")\n return resolved\n # [/DEF:validate_path:Function]\n\n # [DEF:list_files:Function]\n # @PURPOSE: Lists all files and directories in a specific category and subpath.\n # @PARAM: category (Optional[FileCategory]) - The category to list.\n # @PARAM: subpath (Optional[str]) - Nested path within the category.\n # @PARAM: recursive (bool) - Whether to scan nested subdirectories recursively.\n # @PRE: Storage root must exist.\n # @POST: Returns a list of StoredFile objects.\n # @RETURN: List[StoredFile] - List of file and directory metadata objects.\n def list_files(\n self,\n category: Optional[FileCategory] = None,\n subpath: Optional[str] = None,\n recursive: bool = False,\n ) -> List[StoredFile]:\n with belief_scope(\"StoragePlugin:list_files\"):\n root = self.get_storage_root()\n log.reason(\n f\"Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}\"\n )\n files = []\n\n # Root view contract: show category directories only.\n if category is None and not subpath:\n for cat in FileCategory:\n base_dir = root / cat.value\n if not base_dir.exists():\n continue\n stat = base_dir.stat()\n files.append(\n StoredFile(\n name=cat.value,\n path=cat.value,\n size=0,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=cat,\n mime_type=\"directory\",\n )\n )\n return sorted(files, key=lambda x: x.name)\n \n categories = [category] if category else list(FileCategory)\n \n for cat in categories:\n # Scan the category subfolder + optional subpath\n base_dir = root / cat.value\n if subpath:\n target_dir = self.validate_path(base_dir / subpath)\n else:\n target_dir = base_dir\n\n if not target_dir.exists():\n continue\n \n log.reason(f\"Scanning directory: {target_dir}\")\n\n if recursive:\n for current_root, dirs, filenames in os.walk(target_dir):\n dirs[:] = [d for d in dirs if \"Logs\" not in d]\n for filename in filenames:\n file_path = Path(current_root) / filename\n if \"Logs\" in str(file_path):\n continue\n stat = file_path.stat()\n files.append(\n StoredFile(\n name=filename,\n path=str(file_path.relative_to(root)),\n size=stat.st_size,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=cat,\n mime_type=None,\n )\n )\n continue\n\n # Use os.scandir for better performance and to distinguish files vs dirs\n with os.scandir(target_dir) as it:\n for entry in it:\n # Skip logs\n if \"Logs\" in entry.path:\n continue\n\n stat = entry.stat()\n is_dir = entry.is_dir()\n\n files.append(StoredFile(\n name=entry.name,\n path=str(Path(entry.path).relative_to(root)),\n size=stat.st_size if not is_dir else 0,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=cat,\n mime_type=\"directory\" if is_dir else None\n ))\n \n # Sort: directories first, then by name\n return sorted(files, key=lambda x: (x.mime_type != \"directory\", x.name))\n # [/DEF:list_files:Function]\n\n # [DEF:save_file:Function]\n # @PURPOSE: Saves an uploaded file to the specified category and optional subpath.\n # @PARAM: file (UploadFile) - The uploaded file.\n # @PARAM: category (FileCategory) - The target category.\n # @PARAM: subpath (Optional[str]) - The target subpath.\n # @PRE: file must be a valid UploadFile; category must be valid.\n # @POST: File is written to disk and metadata is returned.\n # @RETURN: StoredFile - Metadata of the saved file.\n # @SIDE_EFFECT: Writes file to disk.\n async def save_file(self, file: UploadFile, category: FileCategory, subpath: Optional[str] = None) -> StoredFile:\n with belief_scope(\"StoragePlugin:save_file\"):\n root = self.get_storage_root()\n dest_dir = root / category.value\n if subpath:\n dest_dir = dest_dir / subpath\n \n dest_dir.mkdir(parents=True, exist_ok=True)\n \n dest_path = self.validate_path(dest_dir / file.filename)\n \n with dest_path.open(\"wb\") as buffer:\n shutil.copyfileobj(file.file, buffer)\n \n stat = dest_path.stat()\n return StoredFile(\n name=dest_path.name,\n path=str(dest_path.relative_to(root)),\n size=stat.st_size,\n created_at=datetime.fromtimestamp(stat.st_ctime),\n category=category,\n mime_type=file.content_type\n )\n # [/DEF:save_file:Function]\n\n # [DEF:delete_file:Function]\n # @PURPOSE: Deletes a file or directory from the specified category and path.\n # @PARAM: category (FileCategory) - The category.\n # @PARAM: path (str) - The relative path of the file or directory.\n # @PRE: path must belong to the specified category and exist on disk.\n # @POST: The file or directory is removed from disk.\n # @SIDE_EFFECT: Removes item from disk.\n def delete_file(self, category: FileCategory, path: str):\n with belief_scope(\"StoragePlugin:delete_file\"):\n root = self.get_storage_root()\n # path is relative to root, but we ensure it starts with category\n full_path = self.validate_path(root / path)\n \n if not str(Path(path)).startswith(category.value):\n raise ValueError(f\"Path {path} does not belong to category {category}\")\n\n if full_path.exists():\n if full_path.is_dir():\n shutil.rmtree(full_path)\n else:\n full_path.unlink()\n log.reflect(f\"Deleted: {full_path}\")\n else:\n raise FileNotFoundError(f\"Item {path} not found\")\n # [/DEF:delete_file:Function]\n\n # [DEF:get_file_path:Function]\n # @PURPOSE: Returns the absolute path of a file for download.\n # @PARAM: category (FileCategory) - The category.\n # @PARAM: path (str) - The relative path of the file.\n # @PRE: path must belong to the specified category and be a file.\n # @POST: Returns the absolute Path to the file.\n # @RETURN: Path - Absolute path to the file.\n def get_file_path(self, category: FileCategory, path: str) -> Path:\n with belief_scope(\"StoragePlugin:get_file_path\"):\n root = self.get_storage_root()\n file_path = self.validate_path(root / path)\n \n if not str(Path(path)).startswith(category.value):\n raise ValueError(f\"Path {path} does not belong to category {category}\")\n\n if not file_path.exists() or file_path.is_dir():\n raise FileNotFoundError(f\"File {path} not found\")\n \n return file_path\n # [/DEF:get_file_path:Function]\n\n# [/DEF:StoragePlugin:Class]\n# [/DEF:StoragePlugin:Module]\n" }, { "contract_id": "get_storage_root", "contract_type": "Function", "file_path": "backend/src/plugins/storage/plugin.py", - "start_line": 134, - "end_line": 154, + "start_line": 130, + "end_line": 150, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -62394,8 +63403,8 @@ "contract_id": "resolve_path", "contract_type": "Function", "file_path": "backend/src/plugins/storage/plugin.py", - "start_line": 156, - "end_line": 178, + "start_line": 152, + "end_line": 174, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -62439,14 +63448,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:resolve_path:Function]\n # @PURPOSE: Resolves a dynamic path pattern using provided variables.\n # @PARAM: pattern (str) - The path pattern to resolve.\n # @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.\n # @PRE: pattern must be a valid format string.\n # @POST: Returns the resolved path string.\n # @RETURN: str - The resolved path.\n def resolve_path(self, pattern: str, variables: Dict[str, str]) -> str:\n with belief_scope(\"StoragePlugin:resolve_path\"):\n # Add common variables\n vars_with_defaults = {\n \"timestamp\": datetime.now().strftime(\"%Y%m%dT%H%M%S\"),\n **variables\n }\n try:\n resolved = pattern.format(**vars_with_defaults)\n # Clean up any double slashes or leading/trailing slashes for relative path\n return os.path.normpath(resolved).strip(\"/\")\n except KeyError as e:\n logger.warning(f\"[StoragePlugin][Coherence:Failed] Missing variable for path resolution: {e}\")\n # Fallback to literal pattern if formatting fails partially (or handle as needed)\n return pattern.replace(\"{\", \"\").replace(\"}\", \"\")\n # [/DEF:resolve_path:Function]\n" + "body": " # [DEF:resolve_path:Function]\n # @PURPOSE: Resolves a dynamic path pattern using provided variables.\n # @PARAM: pattern (str) - The path pattern to resolve.\n # @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.\n # @PRE: pattern must be a valid format string.\n # @POST: Returns the resolved path string.\n # @RETURN: str - The resolved path.\n def resolve_path(self, pattern: str, variables: Dict[str, str]) -> str:\n with belief_scope(\"StoragePlugin:resolve_path\"):\n # Add common variables\n vars_with_defaults = {\n \"timestamp\": datetime.now().strftime(\"%Y%m%dT%H%M%S\"),\n **variables\n }\n try:\n resolved = pattern.format(**vars_with_defaults)\n # Clean up any double slashes or leading/trailing slashes for relative path\n return os.path.normpath(resolved).strip(\"/\")\n except KeyError as e:\n log.explore(\"Missing variable for path resolution\", error=str(e))\n # Fallback to literal pattern if formatting fails partially (or handle as needed)\n return pattern.replace(\"{\", \"\").replace(\"}\", \"\")\n # [/DEF:resolve_path:Function]\n" }, { "contract_id": "ensure_directories", "contract_type": "Function", "file_path": "backend/src/plugins/storage/plugin.py", - "start_line": 180, - "end_line": 193, + "start_line": 176, + "end_line": 189, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -62486,14 +63495,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:ensure_directories:Function]\n # @PURPOSE: Creates the storage root and category subdirectories if they don't exist.\n # @PRE: Storage root must be resolvable.\n # @POST: Directories are created on the filesystem.\n # @SIDE_EFFECT: Creates directories on the filesystem.\n def ensure_directories(self):\n with belief_scope(\"StoragePlugin:ensure_directories\"):\n root = self.get_storage_root()\n for category in FileCategory:\n # Use singular name for consistency with BackupPlugin and GitService\n path = root / category.value\n path.mkdir(parents=True, exist_ok=True)\n logger.debug(f\"[StoragePlugin][Action] Ensured directory: {path}\")\n # [/DEF:ensure_directories:Function]\n" + "body": " # [DEF:ensure_directories:Function]\n # @PURPOSE: Creates the storage root and category subdirectories if they don't exist.\n # @PRE: Storage root must be resolvable.\n # @POST: Directories are created on the filesystem.\n # @SIDE_EFFECT: Creates directories on the filesystem.\n def ensure_directories(self):\n with belief_scope(\"StoragePlugin:ensure_directories\"):\n root = self.get_storage_root()\n for category in FileCategory:\n # Use singular name for consistency with BackupPlugin and GitService\n path = root / category.value\n path.mkdir(parents=True, exist_ok=True)\n log.reason(f\"Ensured directory: {path}\")\n # [/DEF:ensure_directories:Function]\n" }, { "contract_id": "save_file", "contract_type": "Function", "file_path": "backend/src/plugins/storage/plugin.py", - "start_line": 309, - "end_line": 341, + "start_line": 305, + "end_line": 337, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -62553,8 +63562,8 @@ "contract_id": "get_file_path", "contract_type": "Function", "file_path": "backend/src/plugins/storage/plugin.py", - "start_line": 369, - "end_line": 388, + "start_line": 365, + "end_line": 384, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -63565,7 +64574,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/translate/dictionary.py", "start_line": 1, - "end_line": 754, + "end_line": 757, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -63648,14 +64657,14 @@ } ], "anchor_syntax": "region", - "body": "# #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter]\n# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TerminologyDictionary]\n# @RELATION DEPENDS_ON -> [DictionaryEntry]\n# @RELATION DEPENDS_ON -> [TranslationJobDictionary]\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]\n# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary.\n# @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.\n# @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links.\n# @REJECTED \"Keep both\" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.\n\nimport csv\nimport io\nimport re\nfrom typing import Any, Dict, List, Optional, Tuple\nfrom sqlalchemy.orm import Session\nfrom sqlalchemy import func\nfrom ...core.logger import logger, belief_scope\nfrom ...models.translate import (\n TerminologyDictionary,\n DictionaryEntry,\n TranslationJobDictionary,\n TranslationJob,\n)\nfrom ._utils import _normalize_term, _detect_delimiter\n\n\n# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]\n# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.\n# @PRE Database session is open and valid.\n# @POST Dictionary and entry mutations are persisted with conflict detection.\n# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.\n# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]\n# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.\nclass DictionaryManager:\n # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]\n # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def create_dictionary(\n db: Session, name: str, source_dialect: str, target_dialect: str,\n created_by: Optional[str] = None, description: Optional[str] = None,\n is_active: bool = True,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.create_dictionary\"):\n logger.reason(\"Creating dictionary\", {\"name\": name, \"source\": source_dialect, \"target\": target_dialect})\n dictionary = TerminologyDictionary(\n name=name,\n description=description,\n source_dialect=source_dialect,\n target_dialect=target_dialect,\n is_active=is_active,\n created_by=created_by,\n )\n db.add(dictionary)\n db.commit()\n db.refresh(dictionary)\n logger.reflect(\"Dictionary created\", {\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.create_dictionary\n\n # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]\n # @BRIEF Update an existing terminology dictionary's metadata fields.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def update_dictionary(\n db: Session, dict_id: str, name: Optional[str] = None,\n description: Optional[str] = None, source_dialect: Optional[str] = None,\n target_dialect: Optional[str] = None, is_active: Optional[bool] = None,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.update_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n logger.reason(\"Updating dictionary\", {\"id\": dict_id})\n if name is not None:\n dictionary.name = name\n if description is not None:\n dictionary.description = description\n if source_dialect is not None:\n dictionary.source_dialect = source_dialect\n if target_dialect is not None:\n dictionary.target_dialect = target_dialect\n if is_active is not None:\n dictionary.is_active = is_active\n db.commit()\n db.refresh(dictionary)\n logger.reflect(\"Dictionary updated\", {\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.update_dictionary\n\n # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]\n # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).\n # @PRE dict_id exists.\n # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.\n # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.\n @staticmethod\n def delete_dictionary(db: Session, dict_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Check for attached active/scheduled jobs\n blocked_statuses = [\"ACTIVE\", \"READY\", \"RUNNING\", \"SCHEDULED\"]\n attached_jobs = (\n db.query(TranslationJobDictionary)\n .filter(\n TranslationJobDictionary.dictionary_id == dict_id,\n TranslationJobDictionary.job_id.in_(\n db.query(TranslationJob.id).filter(\n TranslationJob.status.in_(blocked_statuses)\n )\n ),\n )\n .count()\n )\n\n if attached_jobs > 0:\n logger.explore(\"Delete blocked: dictionary attached to active jobs\", {\"dict_id\": dict_id, \"jobs\": attached_jobs})\n raise ValueError(\n f\"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). \"\n \"Remove dictionary associations from jobs first.\"\n )\n\n logger.reason(\"Deleting dictionary\", {\"id\": dict_id})\n # Delete entries and job-dictionary links first\n db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()\n db.delete(dictionary)\n db.commit()\n logger.reflect(\"Dictionary deleted\", {\"id\": dict_id})\n # #endregion DictionaryManager.delete_dictionary\n\n # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]\n # @BRIEF Get a single dictionary by ID with entry count.\n @staticmethod\n def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n return dictionary\n # #endregion DictionaryManager.get_dictionary\n\n # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]\n # @BRIEF List dictionaries with pagination and total count.\n @staticmethod\n def list_dictionaries(\n db: Session, page: int = 1, page_size: int = 20,\n ) -> Tuple[List[TerminologyDictionary], int]:\n total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0\n dictionaries = (\n db.query(TerminologyDictionary)\n .order_by(TerminologyDictionary.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return dictionaries, total\n # #endregion DictionaryManager.list_dictionaries\n\n # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]\n # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def add_entry(\n db: Session, dict_id: str, source_term: str, target_term: str,\n context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.add_entry\"):\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}). Use overwrite or keep_existing conflict mode.\"\n )\n\n logger.reason(\"Adding dictionary entry\", {\"dict_id\": dict_id, \"term\": source_term})\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=target_term.strip(),\n context_notes=context_notes.strip() if context_notes else None,\n )\n db.add(entry)\n db.commit()\n db.refresh(entry)\n logger.reflect(\"Entry added\", {\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.add_entry\n\n # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]\n # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def edit_entry(\n db: Session, entry_id: str, source_term: Optional[str] = None,\n target_term: Optional[str] = None, context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.edit_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n\n logger.reason(\"Editing dictionary entry\", {\"entry_id\": entry_id})\n if source_term is not None:\n normalized = _normalize_term(source_term)\n # Check uniqueness within the same dictionary\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == entry.dictionary_id,\n DictionaryEntry.source_term_normalized == normalized,\n DictionaryEntry.id != entry_id,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}).\"\n )\n entry.source_term = source_term.strip()\n entry.source_term_normalized = normalized\n if target_term is not None:\n entry.target_term = target_term.strip()\n if context_notes is not None:\n entry.context_notes = context_notes.strip() if context_notes else None\n db.commit()\n db.refresh(entry)\n logger.reflect(\"Entry updated\", {\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.edit_entry\n\n # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]\n # @BRIEF Delete a single dictionary entry.\n @staticmethod\n def delete_entry(db: Session, entry_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n logger.reason(\"Deleting dictionary entry\", {\"entry_id\": entry_id})\n db.delete(entry)\n db.commit()\n logger.reflect(\"Entry deleted\", {\"entry_id\": entry_id})\n # #endregion DictionaryManager.delete_entry\n\n # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]\n # @BRIEF Delete all entries for a dictionary.\n @staticmethod\n def clear_entries(db: Session, dict_id: str) -> int:\n with belief_scope(\"DictionaryManager.clear_entries\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n logger.reason(\"Clearing all entries\", {\"dict_id\": dict_id})\n deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.commit()\n logger.reflect(\"Entries cleared\", {\"dict_id\": dict_id, \"count\": deleted})\n return deleted\n # #endregion DictionaryManager.clear_entries\n\n # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]\n # @BRIEF List entries for a dictionary with pagination.\n @staticmethod\n def list_entries(\n db: Session, dict_id: str, page: int = 1, page_size: int = 50,\n ) -> Tuple[List[DictionaryEntry], int]:\n total = (\n db.query(func.count(DictionaryEntry.id))\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .scalar()\n or 0\n )\n entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .order_by(DictionaryEntry.source_term.asc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return entries, total\n # #endregion DictionaryManager.list_entries\n\n # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]\n # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).\n # @PRE content is valid CSV or TSV. dict_id exists.\n # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.\n # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.\n @staticmethod\n def import_entries(\n db: Session, dict_id: str, content: str,\n delimiter: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n preview_only: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.import_entries\"):\n # Validate dictionary\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Detect delimiter if not specified\n if not delimiter:\n delimiter = _detect_delimiter(content)\n logger.reason(\"Detected delimiter\", {\"delimiter\": repr(delimiter)})\n\n if delimiter not in (\",\", \"\\t\"):\n raise ValueError(f\"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\\\t'.\")\n\n # Parse content\n reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)\n required_fields = {\"source_term\", \"target_term\"}\n if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):\n raise ValueError(\n f\"CSV/TSV must have at least 'source_term' and 'target_term' columns. \"\n f\"Got: {reader.fieldnames}\"\n )\n\n result: Dict[str, Any] = {\n \"total\": 0,\n \"created\": 0,\n \"updated\": 0,\n \"skipped\": 0,\n \"errors\": [],\n \"preview\": [],\n }\n\n rows = list(reader)\n result[\"total\"] = len(rows)\n\n for row_idx, row in enumerate(rows):\n try:\n source_term = row.get(\"source_term\", \"\").strip()\n target_term = row.get(\"target_term\", \"\").strip()\n context_notes = row.get(\"context_notes\", \"\").strip() or None\n\n if not source_term or not target_term:\n result[\"errors\"].append({\n \"line\": row_idx + 2, # +2 for header and 1-indexed\n \"error\": \"Both source_term and target_term are required\",\n \"row\": row,\n })\n continue\n\n normalized = _normalize_term(source_term)\n\n if preview_only:\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n preview_row = {\n \"line\": row_idx + 2,\n \"source_term\": source_term,\n \"target_term\": target_term,\n \"context_notes\": context_notes,\n \"is_conflict\": existing is not None,\n \"existing_target_term\": existing.target_term if existing else None,\n }\n result[\"preview\"].append(preview_row)\n continue\n\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n if on_conflict == \"overwrite\":\n existing.source_term = source_term\n existing.target_term = target_term\n existing.context_notes = context_notes\n result[\"updated\"] += 1\n elif on_conflict == \"keep_existing\":\n result[\"skipped\"] += 1\n else: # cancel\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": f\"Conflict on '{source_term}' and on_conflict='cancel'\",\n \"row\": row,\n })\n continue\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=target_term,\n context_notes=context_notes,\n )\n db.add(entry)\n result[\"created\"] += 1\n\n except Exception as e:\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": str(e),\n \"row\": row,\n })\n\n if not preview_only:\n db.commit()\n\n logger.reflect(\"Import complete\", {\n \"dict_id\": dict_id,\n \"total\": result[\"total\"],\n \"created\": result[\"created\"],\n \"updated\": result[\"updated\"],\n \"skipped\": result[\"skipped\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n # #endregion DictionaryManager.import_entries\n\n # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]\n # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.\n # @PRE job_id exists and source_texts is a list of strings.\n # @POST Returns list of matched entries with match info, sorted by dictionary link priority.\n # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.\n @staticmethod\n def filter_for_batch(\n db: Session, source_texts: List[str], job_id: str,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"DictionaryManager.filter_for_batch\"):\n # Get dictionaries attached to this job\n job_dict_links = (\n db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n if not job_dict_links:\n logger.reason(\"No dictionaries attached to job\", {\"job_id\": job_id})\n return []\n\n dict_ids = [jd.dictionary_id for jd in job_dict_links]\n\n # Get all active dictionaries\n dictionaries = (\n db.query(TerminologyDictionary)\n .filter(\n TerminologyDictionary.id.in_(dict_ids),\n TerminologyDictionary.is_active == True,\n )\n .all()\n )\n if not dictionaries:\n return []\n\n # Build dict_id -> dict_name lookup\n dict_map = {d.id: d.name for d in dictionaries}\n\n # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)\n all_entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id.in_(dict_ids))\n .order_by(DictionaryEntry.source_term.asc())\n .all()\n )\n\n if not all_entries:\n return []\n\n matched: List[Dict[str, Any]] = []\n seen_source_match: set = set()\n\n for text_idx, source_text in enumerate(source_texts):\n if not source_text:\n continue\n\n text_lower = source_text.lower()\n\n for entry in all_entries:\n norm = entry.source_term_normalized\n if not norm:\n continue\n\n # Word-boundary-aware matching\n # Build pattern: \\bterm\\b (case-insensitive)\n # Escape regex special chars in the search term\n escaped = re.escape(norm)\n pattern = re.compile(r\"\\b\" + escaped + r\"\\b\", re.IGNORECASE)\n\n if pattern.search(text_lower):\n match_key = (text_idx, entry.id)\n if match_key not in seen_source_match:\n seen_source_match.add(match_key)\n matched.append({\n \"text_index\": text_idx,\n \"source_text\": source_text,\n \"entry_id\": entry.id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"dictionary_id\": entry.dictionary_id,\n \"dictionary_name\": dict_map.get(entry.dictionary_id, \"\"),\n \"context_notes\": entry.context_notes,\n })\n\n # Sort by dictionary link priority (order of dict_ids from link order)\n dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}\n matched.sort(key=lambda m: (dict_priority.get(m[\"dictionary_id\"], 999), m[\"text_index\"]))\n\n logger.reflect(\"Batch filter match complete\", {\n \"job_id\": job_id,\n \"source_texts\": len(source_texts),\n \"matches\": len(matched),\n \"dictionaries_used\": len(dictionaries),\n })\n return matched\n # #endregion DictionaryManager.filter_for_batch\n\n\n # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]\n # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.\n # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.\n # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.\n # @SIDE_EFFECT Creates/updates DictionaryEntry row.\n @staticmethod\n def submit_correction(\n db: Session,\n dict_id: str,\n source_term: str,\n incorrect_target_term: str,\n corrected_target_term: str,\n origin_run_id: Optional[str] = None,\n origin_row_key: Optional[str] = None,\n origin_user_id: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_correction\"):\n # Validate dictionary exists\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n result: Dict[str, Any] = {\n \"source_term\": source_term,\n \"target_term\": corrected_target_term,\n \"action\": \"created\",\n \"entry_id\": None,\n \"conflict\": None,\n \"message\": None,\n }\n\n if existing:\n # Conflict detected\n if on_conflict == \"keep_existing\":\n result[\"action\"] = \"conflict_detected\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"keep_existing\",\n }\n result[\"message\"] = f\"Existing entry kept: '{existing.target_term}'\"\n return result\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target_term.strip()\n if origin_run_id:\n existing.origin_run_id = origin_run_id\n if origin_row_key:\n existing.origin_row_key = origin_row_key\n if origin_user_id:\n existing.origin_user_id = origin_user_id\n db.flush()\n result[\"action\"] = \"updated\"\n result[\"entry_id\"] = existing.id\n result[\"message\"] = f\"Entry updated from '{existing.target_term}' to '{corrected_target_term}'\"\n else: # cancel\n result[\"action\"] = \"skipped\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"cancel\",\n }\n result[\"message\"] = \"Correction cancelled by conflict mode\"\n return result\n else:\n # Create new entry\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=corrected_target_term.strip(),\n origin_run_id=origin_run_id,\n origin_row_key=origin_row_key,\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n db.flush()\n result[\"entry_id\"] = entry.id\n result[\"message\"] = f\"Entry created for '{source_term}' -> '{corrected_target_term}'\"\n\n db.commit()\n logger.reflect(\"Correction processed\", result)\n return result\n # #endregion DictionaryManager.submit_correction\n\n # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]\n # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.\n # @PRE corrections list is non-empty. dict_id exists.\n # @POST All corrections applied or none applied with conflict list (atomic).\n # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.\n @staticmethod\n def submit_bulk_corrections(\n db: Session,\n dict_id: str,\n corrections: List[Dict[str, Any]],\n origin_user_id: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_bulk_corrections\"):\n # Validate dictionary first\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n results: List[Dict[str, Any]] = []\n conflicts: List[Dict[str, Any]] = []\n all_ok = True\n\n for corr in corrections:\n source_term = corr.get(\"source_term\", \"\").strip()\n incorrect_target = corr.get(\"incorrect_target_term\", \"\").strip()\n corrected_target = corr.get(\"corrected_target_term\", \"\").strip()\n\n if not source_term or not corrected_target:\n results.append({\n \"source_term\": source_term,\n \"action\": \"error\",\n \"message\": \"source_term and corrected_target_term are required\",\n })\n all_ok = False\n continue\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n on_conflict = corr.get(\"on_conflict\", \"overwrite\")\n if on_conflict == \"keep_existing\":\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"keep_existing\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"conflict_detected\",\n \"message\": f\"Existing entry kept: '{existing.target_term}'\",\n })\n continue\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target\n existing.origin_user_id = origin_user_id\n results.append({\n \"source_term\": source_term,\n \"action\": \"updated\",\n \"entry_id\": existing.id,\n \"message\": f\"Updated to '{corrected_target}'\",\n })\n else:\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"cancel\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"skipped\",\n \"message\": \"Cancelled by conflict mode\",\n })\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=corrected_target,\n origin_run_id=corr.get(\"origin_run_id\"),\n origin_row_key=corr.get(\"origin_row_key\"),\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n results.append({\n \"source_term\": source_term,\n \"action\": \"created\",\n \"message\": f\"Entry created for '{source_term}' -> '{corrected_target}'\",\n })\n\n if conflicts and not all_ok:\n # Rollback — all failed\n db.rollback()\n return {\n \"status\": \"conflicts\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"message\": \"Conflicts detected. Resolve before retrying.\",\n }\n\n db.commit()\n\n return {\n \"status\": \"completed\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"total\": len(corrections),\n \"applied\": sum(1 for r in results if r[\"action\"] in (\"created\", \"updated\")),\n }\n # #endregion DictionaryManager.submit_bulk_corrections\n\n\n# #endregion DictionaryManager\n# #endregion DictionaryManagerModule\n" + "body": "# #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter]\n# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TerminologyDictionary]\n# @RELATION DEPENDS_ON -> [DictionaryEntry]\n# @RELATION DEPENDS_ON -> [TranslationJobDictionary]\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]\n# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary.\n# @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.\n# @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links.\n# @REJECTED \"Keep both\" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.\n\nimport csv\nimport io\nimport re\nfrom typing import Any, Dict, List, Optional, Tuple\nfrom sqlalchemy.orm import Session\nfrom sqlalchemy import func\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.logger import belief_scope\nfrom ...models.translate import (\n TerminologyDictionary,\n DictionaryEntry,\n TranslationJobDictionary,\n TranslationJob,\n)\nfrom ._utils import _normalize_term, _detect_delimiter\n\nlog = MarkerLogger(\"DictionaryManager\")\n\n\n# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]\n# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.\n# @PRE Database session is open and valid.\n# @POST Dictionary and entry mutations are persisted with conflict detection.\n# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.\n# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]\n# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.\nclass DictionaryManager:\n # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]\n # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def create_dictionary(\n db: Session, name: str, source_dialect: str, target_dialect: str,\n created_by: Optional[str] = None, description: Optional[str] = None,\n is_active: bool = True,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.create_dictionary\"):\n log.reason(\"Creating dictionary\", payload={\"name\": name, \"source\": source_dialect, \"target\": target_dialect})\n dictionary = TerminologyDictionary(\n name=name,\n description=description,\n source_dialect=source_dialect,\n target_dialect=target_dialect,\n is_active=is_active,\n created_by=created_by,\n )\n db.add(dictionary)\n db.commit()\n db.refresh(dictionary)\n log.reflect(\"Dictionary created\", payload={\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.create_dictionary\n\n # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]\n # @BRIEF Update an existing terminology dictionary's metadata fields.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def update_dictionary(\n db: Session, dict_id: str, name: Optional[str] = None,\n description: Optional[str] = None, source_dialect: Optional[str] = None,\n target_dialect: Optional[str] = None, is_active: Optional[bool] = None,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.update_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n log.reason(\"Updating dictionary\", payload={\"id\": dict_id})\n if name is not None:\n dictionary.name = name\n if description is not None:\n dictionary.description = description\n if source_dialect is not None:\n dictionary.source_dialect = source_dialect\n if target_dialect is not None:\n dictionary.target_dialect = target_dialect\n if is_active is not None:\n dictionary.is_active = is_active\n db.commit()\n db.refresh(dictionary)\n log.reflect(\"Dictionary updated\", payload={\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.update_dictionary\n\n # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]\n # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).\n # @PRE dict_id exists.\n # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.\n # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.\n @staticmethod\n def delete_dictionary(db: Session, dict_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Check for attached active/scheduled jobs\n blocked_statuses = [\"ACTIVE\", \"READY\", \"RUNNING\", \"SCHEDULED\"]\n attached_jobs = (\n db.query(TranslationJobDictionary)\n .filter(\n TranslationJobDictionary.dictionary_id == dict_id,\n TranslationJobDictionary.job_id.in_(\n db.query(TranslationJob.id).filter(\n TranslationJob.status.in_(blocked_statuses)\n )\n ),\n )\n .count()\n )\n\n if attached_jobs > 0:\n log.explore(\"Delete blocked: dictionary attached to active jobs\", payload={\"dict_id\": dict_id, \"jobs\": attached_jobs}, error=f\"Dictionary attached to {attached_jobs} active job(s)\")\n raise ValueError(\n f\"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). \"\n \"Remove dictionary associations from jobs first.\"\n )\n\n log.reason(\"Deleting dictionary\", payload={\"id\": dict_id})\n # Delete entries and job-dictionary links first\n db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()\n db.delete(dictionary)\n db.commit()\n log.reflect(\"Dictionary deleted\", payload={\"id\": dict_id})\n # #endregion DictionaryManager.delete_dictionary\n\n # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]\n # @BRIEF Get a single dictionary by ID with entry count.\n @staticmethod\n def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n return dictionary\n # #endregion DictionaryManager.get_dictionary\n\n # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]\n # @BRIEF List dictionaries with pagination and total count.\n @staticmethod\n def list_dictionaries(\n db: Session, page: int = 1, page_size: int = 20,\n ) -> Tuple[List[TerminologyDictionary], int]:\n total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0\n dictionaries = (\n db.query(TerminologyDictionary)\n .order_by(TerminologyDictionary.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return dictionaries, total\n # #endregion DictionaryManager.list_dictionaries\n\n # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]\n # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def add_entry(\n db: Session, dict_id: str, source_term: str, target_term: str,\n context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.add_entry\"):\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}). Use overwrite or keep_existing conflict mode.\"\n )\n\n log.reason(\"Adding dictionary entry\", payload={\"dict_id\": dict_id, \"term\": source_term})\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=target_term.strip(),\n context_notes=context_notes.strip() if context_notes else None,\n )\n db.add(entry)\n db.commit()\n db.refresh(entry)\n log.reflect(\"Entry added\", payload={\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.add_entry\n\n # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]\n # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def edit_entry(\n db: Session, entry_id: str, source_term: Optional[str] = None,\n target_term: Optional[str] = None, context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.edit_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n\n log.reason(\"Editing dictionary entry\", payload={\"entry_id\": entry_id})\n if source_term is not None:\n normalized = _normalize_term(source_term)\n # Check uniqueness within the same dictionary\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == entry.dictionary_id,\n DictionaryEntry.source_term_normalized == normalized,\n DictionaryEntry.id != entry_id,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}).\"\n )\n entry.source_term = source_term.strip()\n entry.source_term_normalized = normalized\n if target_term is not None:\n entry.target_term = target_term.strip()\n if context_notes is not None:\n entry.context_notes = context_notes.strip() if context_notes else None\n db.commit()\n db.refresh(entry)\n log.reflect(\"Entry updated\", payload={\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.edit_entry\n\n # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]\n # @BRIEF Delete a single dictionary entry.\n @staticmethod\n def delete_entry(db: Session, entry_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n log.reason(\"Deleting dictionary entry\", payload={\"entry_id\": entry_id})\n db.delete(entry)\n db.commit()\n log.reflect(\"Entry deleted\", payload={\"entry_id\": entry_id})\n # #endregion DictionaryManager.delete_entry\n\n # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]\n # @BRIEF Delete all entries for a dictionary.\n @staticmethod\n def clear_entries(db: Session, dict_id: str) -> int:\n with belief_scope(\"DictionaryManager.clear_entries\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n log.reason(\"Clearing all entries\", payload={\"dict_id\": dict_id})\n deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.commit()\n log.reflect(\"Entries cleared\", payload={\"dict_id\": dict_id, \"count\": deleted})\n return deleted\n # #endregion DictionaryManager.clear_entries\n\n # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]\n # @BRIEF List entries for a dictionary with pagination.\n @staticmethod\n def list_entries(\n db: Session, dict_id: str, page: int = 1, page_size: int = 50,\n ) -> Tuple[List[DictionaryEntry], int]:\n total = (\n db.query(func.count(DictionaryEntry.id))\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .scalar()\n or 0\n )\n entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .order_by(DictionaryEntry.source_term.asc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return entries, total\n # #endregion DictionaryManager.list_entries\n\n # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]\n # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).\n # @PRE content is valid CSV or TSV. dict_id exists.\n # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.\n # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.\n @staticmethod\n def import_entries(\n db: Session, dict_id: str, content: str,\n delimiter: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n preview_only: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.import_entries\"):\n # Validate dictionary\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Detect delimiter if not specified\n if not delimiter:\n delimiter = _detect_delimiter(content)\n log.reason(\"Detected delimiter\", payload={\"delimiter\": repr(delimiter)})\n\n if delimiter not in (\",\", \"\\t\"):\n raise ValueError(f\"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\\\t'.\")\n\n # Parse content\n reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)\n required_fields = {\"source_term\", \"target_term\"}\n if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):\n raise ValueError(\n f\"CSV/TSV must have at least 'source_term' and 'target_term' columns. \"\n f\"Got: {reader.fieldnames}\"\n )\n\n result: Dict[str, Any] = {\n \"total\": 0,\n \"created\": 0,\n \"updated\": 0,\n \"skipped\": 0,\n \"errors\": [],\n \"preview\": [],\n }\n\n rows = list(reader)\n result[\"total\"] = len(rows)\n\n for row_idx, row in enumerate(rows):\n try:\n source_term = row.get(\"source_term\", \"\").strip()\n target_term = row.get(\"target_term\", \"\").strip()\n context_notes = row.get(\"context_notes\", \"\").strip() or None\n\n if not source_term or not target_term:\n result[\"errors\"].append({\n \"line\": row_idx + 2, # +2 for header and 1-indexed\n \"error\": \"Both source_term and target_term are required\",\n \"row\": row,\n })\n continue\n\n normalized = _normalize_term(source_term)\n\n if preview_only:\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n preview_row = {\n \"line\": row_idx + 2,\n \"source_term\": source_term,\n \"target_term\": target_term,\n \"context_notes\": context_notes,\n \"is_conflict\": existing is not None,\n \"existing_target_term\": existing.target_term if existing else None,\n }\n result[\"preview\"].append(preview_row)\n continue\n\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n if on_conflict == \"overwrite\":\n existing.source_term = source_term\n existing.target_term = target_term\n existing.context_notes = context_notes\n result[\"updated\"] += 1\n elif on_conflict == \"keep_existing\":\n result[\"skipped\"] += 1\n else: # cancel\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": f\"Conflict on '{source_term}' and on_conflict='cancel'\",\n \"row\": row,\n })\n continue\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=target_term,\n context_notes=context_notes,\n )\n db.add(entry)\n result[\"created\"] += 1\n\n except Exception as e:\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": str(e),\n \"row\": row,\n })\n\n if not preview_only:\n db.commit()\n\n log.reflect(\"Import complete\", payload={\n \"dict_id\": dict_id,\n \"total\": result[\"total\"],\n \"created\": result[\"created\"],\n \"updated\": result[\"updated\"],\n \"skipped\": result[\"skipped\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n # #endregion DictionaryManager.import_entries\n\n # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]\n # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.\n # @PRE job_id exists and source_texts is a list of strings.\n # @POST Returns list of matched entries with match info, sorted by dictionary link priority.\n # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.\n @staticmethod\n def filter_for_batch(\n db: Session, source_texts: List[str], job_id: str,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"DictionaryManager.filter_for_batch\"):\n # Get dictionaries attached to this job\n job_dict_links = (\n db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n if not job_dict_links:\n log.reason(\"No dictionaries attached to job\", payload={\"job_id\": job_id})\n return []\n\n dict_ids = [jd.dictionary_id for jd in job_dict_links]\n\n # Get all active dictionaries\n dictionaries = (\n db.query(TerminologyDictionary)\n .filter(\n TerminologyDictionary.id.in_(dict_ids),\n TerminologyDictionary.is_active == True,\n )\n .all()\n )\n if not dictionaries:\n return []\n\n # Build dict_id -> dict_name lookup\n dict_map = {d.id: d.name for d in dictionaries}\n\n # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)\n all_entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id.in_(dict_ids))\n .order_by(DictionaryEntry.source_term.asc())\n .all()\n )\n\n if not all_entries:\n return []\n\n matched: List[Dict[str, Any]] = []\n seen_source_match: set = set()\n\n for text_idx, source_text in enumerate(source_texts):\n if not source_text:\n continue\n\n text_lower = source_text.lower()\n\n for entry in all_entries:\n norm = entry.source_term_normalized\n if not norm:\n continue\n\n # Word-boundary-aware matching\n # Build pattern: \\bterm\\b (case-insensitive)\n # Escape regex special chars in the search term\n escaped = re.escape(norm)\n pattern = re.compile(r\"\\b\" + escaped + r\"\\b\", re.IGNORECASE)\n\n if pattern.search(text_lower):\n match_key = (text_idx, entry.id)\n if match_key not in seen_source_match:\n seen_source_match.add(match_key)\n matched.append({\n \"text_index\": text_idx,\n \"source_text\": source_text,\n \"entry_id\": entry.id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"dictionary_id\": entry.dictionary_id,\n \"dictionary_name\": dict_map.get(entry.dictionary_id, \"\"),\n \"context_notes\": entry.context_notes,\n })\n\n # Sort by dictionary link priority (order of dict_ids from link order)\n dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}\n matched.sort(key=lambda m: (dict_priority.get(m[\"dictionary_id\"], 999), m[\"text_index\"]))\n\n log.reflect(\"Batch filter match complete\", payload={\n \"job_id\": job_id,\n \"source_texts\": len(source_texts),\n \"matches\": len(matched),\n \"dictionaries_used\": len(dictionaries),\n })\n return matched\n # #endregion DictionaryManager.filter_for_batch\n\n\n # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]\n # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.\n # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.\n # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.\n # @SIDE_EFFECT Creates/updates DictionaryEntry row.\n @staticmethod\n def submit_correction(\n db: Session,\n dict_id: str,\n source_term: str,\n incorrect_target_term: str,\n corrected_target_term: str,\n origin_run_id: Optional[str] = None,\n origin_row_key: Optional[str] = None,\n origin_user_id: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_correction\"):\n # Validate dictionary exists\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n result: Dict[str, Any] = {\n \"source_term\": source_term,\n \"target_term\": corrected_target_term,\n \"action\": \"created\",\n \"entry_id\": None,\n \"conflict\": None,\n \"message\": None,\n }\n\n if existing:\n # Conflict detected\n if on_conflict == \"keep_existing\":\n result[\"action\"] = \"conflict_detected\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"keep_existing\",\n }\n result[\"message\"] = f\"Existing entry kept: '{existing.target_term}'\"\n return result\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target_term.strip()\n if origin_run_id:\n existing.origin_run_id = origin_run_id\n if origin_row_key:\n existing.origin_row_key = origin_row_key\n if origin_user_id:\n existing.origin_user_id = origin_user_id\n db.flush()\n result[\"action\"] = \"updated\"\n result[\"entry_id\"] = existing.id\n result[\"message\"] = f\"Entry updated from '{existing.target_term}' to '{corrected_target_term}'\"\n else: # cancel\n result[\"action\"] = \"skipped\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"cancel\",\n }\n result[\"message\"] = \"Correction cancelled by conflict mode\"\n return result\n else:\n # Create new entry\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=corrected_target_term.strip(),\n origin_run_id=origin_run_id,\n origin_row_key=origin_row_key,\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n db.flush()\n result[\"entry_id\"] = entry.id\n result[\"message\"] = f\"Entry created for '{source_term}' -> '{corrected_target_term}'\"\n\n db.commit()\n log.reflect(\"Correction processed\", payload=result)\n return result\n # #endregion DictionaryManager.submit_correction\n\n # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]\n # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.\n # @PRE corrections list is non-empty. dict_id exists.\n # @POST All corrections applied or none applied with conflict list (atomic).\n # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.\n @staticmethod\n def submit_bulk_corrections(\n db: Session,\n dict_id: str,\n corrections: List[Dict[str, Any]],\n origin_user_id: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_bulk_corrections\"):\n # Validate dictionary first\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n results: List[Dict[str, Any]] = []\n conflicts: List[Dict[str, Any]] = []\n all_ok = True\n\n for corr in corrections:\n source_term = corr.get(\"source_term\", \"\").strip()\n incorrect_target = corr.get(\"incorrect_target_term\", \"\").strip()\n corrected_target = corr.get(\"corrected_target_term\", \"\").strip()\n\n if not source_term or not corrected_target:\n results.append({\n \"source_term\": source_term,\n \"action\": \"error\",\n \"message\": \"source_term and corrected_target_term are required\",\n })\n all_ok = False\n continue\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n on_conflict = corr.get(\"on_conflict\", \"overwrite\")\n if on_conflict == \"keep_existing\":\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"keep_existing\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"conflict_detected\",\n \"message\": f\"Existing entry kept: '{existing.target_term}'\",\n })\n continue\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target\n existing.origin_user_id = origin_user_id\n results.append({\n \"source_term\": source_term,\n \"action\": \"updated\",\n \"entry_id\": existing.id,\n \"message\": f\"Updated to '{corrected_target}'\",\n })\n else:\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"cancel\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"skipped\",\n \"message\": \"Cancelled by conflict mode\",\n })\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=corrected_target,\n origin_run_id=corr.get(\"origin_run_id\"),\n origin_row_key=corr.get(\"origin_row_key\"),\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n results.append({\n \"source_term\": source_term,\n \"action\": \"created\",\n \"message\": f\"Entry created for '{source_term}' -> '{corrected_target}'\",\n })\n\n if conflicts and not all_ok:\n # Rollback — all failed\n db.rollback()\n return {\n \"status\": \"conflicts\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"message\": \"Conflicts detected. Resolve before retrying.\",\n }\n\n db.commit()\n\n return {\n \"status\": \"completed\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"total\": len(corrections),\n \"applied\": sum(1 for r in results if r[\"action\"] in (\"created\", \"updated\")),\n }\n # #endregion DictionaryManager.submit_bulk_corrections\n\n\n# #endregion DictionaryManager\n# #endregion DictionaryManagerModule\n" }, { "contract_id": "DictionaryManager", "contract_type": "Class", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 30, - "end_line": 753, + "start_line": 33, + "end_line": 756, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -63695,14 +64704,14 @@ } ], "anchor_syntax": "region", - "body": "# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]\n# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.\n# @PRE Database session is open and valid.\n# @POST Dictionary and entry mutations are persisted with conflict detection.\n# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.\n# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]\n# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.\nclass DictionaryManager:\n # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]\n # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def create_dictionary(\n db: Session, name: str, source_dialect: str, target_dialect: str,\n created_by: Optional[str] = None, description: Optional[str] = None,\n is_active: bool = True,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.create_dictionary\"):\n logger.reason(\"Creating dictionary\", {\"name\": name, \"source\": source_dialect, \"target\": target_dialect})\n dictionary = TerminologyDictionary(\n name=name,\n description=description,\n source_dialect=source_dialect,\n target_dialect=target_dialect,\n is_active=is_active,\n created_by=created_by,\n )\n db.add(dictionary)\n db.commit()\n db.refresh(dictionary)\n logger.reflect(\"Dictionary created\", {\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.create_dictionary\n\n # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]\n # @BRIEF Update an existing terminology dictionary's metadata fields.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def update_dictionary(\n db: Session, dict_id: str, name: Optional[str] = None,\n description: Optional[str] = None, source_dialect: Optional[str] = None,\n target_dialect: Optional[str] = None, is_active: Optional[bool] = None,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.update_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n logger.reason(\"Updating dictionary\", {\"id\": dict_id})\n if name is not None:\n dictionary.name = name\n if description is not None:\n dictionary.description = description\n if source_dialect is not None:\n dictionary.source_dialect = source_dialect\n if target_dialect is not None:\n dictionary.target_dialect = target_dialect\n if is_active is not None:\n dictionary.is_active = is_active\n db.commit()\n db.refresh(dictionary)\n logger.reflect(\"Dictionary updated\", {\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.update_dictionary\n\n # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]\n # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).\n # @PRE dict_id exists.\n # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.\n # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.\n @staticmethod\n def delete_dictionary(db: Session, dict_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Check for attached active/scheduled jobs\n blocked_statuses = [\"ACTIVE\", \"READY\", \"RUNNING\", \"SCHEDULED\"]\n attached_jobs = (\n db.query(TranslationJobDictionary)\n .filter(\n TranslationJobDictionary.dictionary_id == dict_id,\n TranslationJobDictionary.job_id.in_(\n db.query(TranslationJob.id).filter(\n TranslationJob.status.in_(blocked_statuses)\n )\n ),\n )\n .count()\n )\n\n if attached_jobs > 0:\n logger.explore(\"Delete blocked: dictionary attached to active jobs\", {\"dict_id\": dict_id, \"jobs\": attached_jobs})\n raise ValueError(\n f\"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). \"\n \"Remove dictionary associations from jobs first.\"\n )\n\n logger.reason(\"Deleting dictionary\", {\"id\": dict_id})\n # Delete entries and job-dictionary links first\n db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()\n db.delete(dictionary)\n db.commit()\n logger.reflect(\"Dictionary deleted\", {\"id\": dict_id})\n # #endregion DictionaryManager.delete_dictionary\n\n # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]\n # @BRIEF Get a single dictionary by ID with entry count.\n @staticmethod\n def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n return dictionary\n # #endregion DictionaryManager.get_dictionary\n\n # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]\n # @BRIEF List dictionaries with pagination and total count.\n @staticmethod\n def list_dictionaries(\n db: Session, page: int = 1, page_size: int = 20,\n ) -> Tuple[List[TerminologyDictionary], int]:\n total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0\n dictionaries = (\n db.query(TerminologyDictionary)\n .order_by(TerminologyDictionary.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return dictionaries, total\n # #endregion DictionaryManager.list_dictionaries\n\n # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]\n # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def add_entry(\n db: Session, dict_id: str, source_term: str, target_term: str,\n context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.add_entry\"):\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}). Use overwrite or keep_existing conflict mode.\"\n )\n\n logger.reason(\"Adding dictionary entry\", {\"dict_id\": dict_id, \"term\": source_term})\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=target_term.strip(),\n context_notes=context_notes.strip() if context_notes else None,\n )\n db.add(entry)\n db.commit()\n db.refresh(entry)\n logger.reflect(\"Entry added\", {\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.add_entry\n\n # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]\n # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def edit_entry(\n db: Session, entry_id: str, source_term: Optional[str] = None,\n target_term: Optional[str] = None, context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.edit_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n\n logger.reason(\"Editing dictionary entry\", {\"entry_id\": entry_id})\n if source_term is not None:\n normalized = _normalize_term(source_term)\n # Check uniqueness within the same dictionary\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == entry.dictionary_id,\n DictionaryEntry.source_term_normalized == normalized,\n DictionaryEntry.id != entry_id,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}).\"\n )\n entry.source_term = source_term.strip()\n entry.source_term_normalized = normalized\n if target_term is not None:\n entry.target_term = target_term.strip()\n if context_notes is not None:\n entry.context_notes = context_notes.strip() if context_notes else None\n db.commit()\n db.refresh(entry)\n logger.reflect(\"Entry updated\", {\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.edit_entry\n\n # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]\n # @BRIEF Delete a single dictionary entry.\n @staticmethod\n def delete_entry(db: Session, entry_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n logger.reason(\"Deleting dictionary entry\", {\"entry_id\": entry_id})\n db.delete(entry)\n db.commit()\n logger.reflect(\"Entry deleted\", {\"entry_id\": entry_id})\n # #endregion DictionaryManager.delete_entry\n\n # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]\n # @BRIEF Delete all entries for a dictionary.\n @staticmethod\n def clear_entries(db: Session, dict_id: str) -> int:\n with belief_scope(\"DictionaryManager.clear_entries\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n logger.reason(\"Clearing all entries\", {\"dict_id\": dict_id})\n deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.commit()\n logger.reflect(\"Entries cleared\", {\"dict_id\": dict_id, \"count\": deleted})\n return deleted\n # #endregion DictionaryManager.clear_entries\n\n # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]\n # @BRIEF List entries for a dictionary with pagination.\n @staticmethod\n def list_entries(\n db: Session, dict_id: str, page: int = 1, page_size: int = 50,\n ) -> Tuple[List[DictionaryEntry], int]:\n total = (\n db.query(func.count(DictionaryEntry.id))\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .scalar()\n or 0\n )\n entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .order_by(DictionaryEntry.source_term.asc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return entries, total\n # #endregion DictionaryManager.list_entries\n\n # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]\n # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).\n # @PRE content is valid CSV or TSV. dict_id exists.\n # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.\n # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.\n @staticmethod\n def import_entries(\n db: Session, dict_id: str, content: str,\n delimiter: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n preview_only: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.import_entries\"):\n # Validate dictionary\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Detect delimiter if not specified\n if not delimiter:\n delimiter = _detect_delimiter(content)\n logger.reason(\"Detected delimiter\", {\"delimiter\": repr(delimiter)})\n\n if delimiter not in (\",\", \"\\t\"):\n raise ValueError(f\"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\\\t'.\")\n\n # Parse content\n reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)\n required_fields = {\"source_term\", \"target_term\"}\n if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):\n raise ValueError(\n f\"CSV/TSV must have at least 'source_term' and 'target_term' columns. \"\n f\"Got: {reader.fieldnames}\"\n )\n\n result: Dict[str, Any] = {\n \"total\": 0,\n \"created\": 0,\n \"updated\": 0,\n \"skipped\": 0,\n \"errors\": [],\n \"preview\": [],\n }\n\n rows = list(reader)\n result[\"total\"] = len(rows)\n\n for row_idx, row in enumerate(rows):\n try:\n source_term = row.get(\"source_term\", \"\").strip()\n target_term = row.get(\"target_term\", \"\").strip()\n context_notes = row.get(\"context_notes\", \"\").strip() or None\n\n if not source_term or not target_term:\n result[\"errors\"].append({\n \"line\": row_idx + 2, # +2 for header and 1-indexed\n \"error\": \"Both source_term and target_term are required\",\n \"row\": row,\n })\n continue\n\n normalized = _normalize_term(source_term)\n\n if preview_only:\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n preview_row = {\n \"line\": row_idx + 2,\n \"source_term\": source_term,\n \"target_term\": target_term,\n \"context_notes\": context_notes,\n \"is_conflict\": existing is not None,\n \"existing_target_term\": existing.target_term if existing else None,\n }\n result[\"preview\"].append(preview_row)\n continue\n\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n if on_conflict == \"overwrite\":\n existing.source_term = source_term\n existing.target_term = target_term\n existing.context_notes = context_notes\n result[\"updated\"] += 1\n elif on_conflict == \"keep_existing\":\n result[\"skipped\"] += 1\n else: # cancel\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": f\"Conflict on '{source_term}' and on_conflict='cancel'\",\n \"row\": row,\n })\n continue\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=target_term,\n context_notes=context_notes,\n )\n db.add(entry)\n result[\"created\"] += 1\n\n except Exception as e:\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": str(e),\n \"row\": row,\n })\n\n if not preview_only:\n db.commit()\n\n logger.reflect(\"Import complete\", {\n \"dict_id\": dict_id,\n \"total\": result[\"total\"],\n \"created\": result[\"created\"],\n \"updated\": result[\"updated\"],\n \"skipped\": result[\"skipped\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n # #endregion DictionaryManager.import_entries\n\n # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]\n # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.\n # @PRE job_id exists and source_texts is a list of strings.\n # @POST Returns list of matched entries with match info, sorted by dictionary link priority.\n # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.\n @staticmethod\n def filter_for_batch(\n db: Session, source_texts: List[str], job_id: str,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"DictionaryManager.filter_for_batch\"):\n # Get dictionaries attached to this job\n job_dict_links = (\n db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n if not job_dict_links:\n logger.reason(\"No dictionaries attached to job\", {\"job_id\": job_id})\n return []\n\n dict_ids = [jd.dictionary_id for jd in job_dict_links]\n\n # Get all active dictionaries\n dictionaries = (\n db.query(TerminologyDictionary)\n .filter(\n TerminologyDictionary.id.in_(dict_ids),\n TerminologyDictionary.is_active == True,\n )\n .all()\n )\n if not dictionaries:\n return []\n\n # Build dict_id -> dict_name lookup\n dict_map = {d.id: d.name for d in dictionaries}\n\n # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)\n all_entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id.in_(dict_ids))\n .order_by(DictionaryEntry.source_term.asc())\n .all()\n )\n\n if not all_entries:\n return []\n\n matched: List[Dict[str, Any]] = []\n seen_source_match: set = set()\n\n for text_idx, source_text in enumerate(source_texts):\n if not source_text:\n continue\n\n text_lower = source_text.lower()\n\n for entry in all_entries:\n norm = entry.source_term_normalized\n if not norm:\n continue\n\n # Word-boundary-aware matching\n # Build pattern: \\bterm\\b (case-insensitive)\n # Escape regex special chars in the search term\n escaped = re.escape(norm)\n pattern = re.compile(r\"\\b\" + escaped + r\"\\b\", re.IGNORECASE)\n\n if pattern.search(text_lower):\n match_key = (text_idx, entry.id)\n if match_key not in seen_source_match:\n seen_source_match.add(match_key)\n matched.append({\n \"text_index\": text_idx,\n \"source_text\": source_text,\n \"entry_id\": entry.id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"dictionary_id\": entry.dictionary_id,\n \"dictionary_name\": dict_map.get(entry.dictionary_id, \"\"),\n \"context_notes\": entry.context_notes,\n })\n\n # Sort by dictionary link priority (order of dict_ids from link order)\n dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}\n matched.sort(key=lambda m: (dict_priority.get(m[\"dictionary_id\"], 999), m[\"text_index\"]))\n\n logger.reflect(\"Batch filter match complete\", {\n \"job_id\": job_id,\n \"source_texts\": len(source_texts),\n \"matches\": len(matched),\n \"dictionaries_used\": len(dictionaries),\n })\n return matched\n # #endregion DictionaryManager.filter_for_batch\n\n\n # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]\n # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.\n # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.\n # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.\n # @SIDE_EFFECT Creates/updates DictionaryEntry row.\n @staticmethod\n def submit_correction(\n db: Session,\n dict_id: str,\n source_term: str,\n incorrect_target_term: str,\n corrected_target_term: str,\n origin_run_id: Optional[str] = None,\n origin_row_key: Optional[str] = None,\n origin_user_id: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_correction\"):\n # Validate dictionary exists\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n result: Dict[str, Any] = {\n \"source_term\": source_term,\n \"target_term\": corrected_target_term,\n \"action\": \"created\",\n \"entry_id\": None,\n \"conflict\": None,\n \"message\": None,\n }\n\n if existing:\n # Conflict detected\n if on_conflict == \"keep_existing\":\n result[\"action\"] = \"conflict_detected\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"keep_existing\",\n }\n result[\"message\"] = f\"Existing entry kept: '{existing.target_term}'\"\n return result\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target_term.strip()\n if origin_run_id:\n existing.origin_run_id = origin_run_id\n if origin_row_key:\n existing.origin_row_key = origin_row_key\n if origin_user_id:\n existing.origin_user_id = origin_user_id\n db.flush()\n result[\"action\"] = \"updated\"\n result[\"entry_id\"] = existing.id\n result[\"message\"] = f\"Entry updated from '{existing.target_term}' to '{corrected_target_term}'\"\n else: # cancel\n result[\"action\"] = \"skipped\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"cancel\",\n }\n result[\"message\"] = \"Correction cancelled by conflict mode\"\n return result\n else:\n # Create new entry\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=corrected_target_term.strip(),\n origin_run_id=origin_run_id,\n origin_row_key=origin_row_key,\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n db.flush()\n result[\"entry_id\"] = entry.id\n result[\"message\"] = f\"Entry created for '{source_term}' -> '{corrected_target_term}'\"\n\n db.commit()\n logger.reflect(\"Correction processed\", result)\n return result\n # #endregion DictionaryManager.submit_correction\n\n # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]\n # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.\n # @PRE corrections list is non-empty. dict_id exists.\n # @POST All corrections applied or none applied with conflict list (atomic).\n # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.\n @staticmethod\n def submit_bulk_corrections(\n db: Session,\n dict_id: str,\n corrections: List[Dict[str, Any]],\n origin_user_id: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_bulk_corrections\"):\n # Validate dictionary first\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n results: List[Dict[str, Any]] = []\n conflicts: List[Dict[str, Any]] = []\n all_ok = True\n\n for corr in corrections:\n source_term = corr.get(\"source_term\", \"\").strip()\n incorrect_target = corr.get(\"incorrect_target_term\", \"\").strip()\n corrected_target = corr.get(\"corrected_target_term\", \"\").strip()\n\n if not source_term or not corrected_target:\n results.append({\n \"source_term\": source_term,\n \"action\": \"error\",\n \"message\": \"source_term and corrected_target_term are required\",\n })\n all_ok = False\n continue\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n on_conflict = corr.get(\"on_conflict\", \"overwrite\")\n if on_conflict == \"keep_existing\":\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"keep_existing\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"conflict_detected\",\n \"message\": f\"Existing entry kept: '{existing.target_term}'\",\n })\n continue\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target\n existing.origin_user_id = origin_user_id\n results.append({\n \"source_term\": source_term,\n \"action\": \"updated\",\n \"entry_id\": existing.id,\n \"message\": f\"Updated to '{corrected_target}'\",\n })\n else:\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"cancel\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"skipped\",\n \"message\": \"Cancelled by conflict mode\",\n })\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=corrected_target,\n origin_run_id=corr.get(\"origin_run_id\"),\n origin_row_key=corr.get(\"origin_row_key\"),\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n results.append({\n \"source_term\": source_term,\n \"action\": \"created\",\n \"message\": f\"Entry created for '{source_term}' -> '{corrected_target}'\",\n })\n\n if conflicts and not all_ok:\n # Rollback — all failed\n db.rollback()\n return {\n \"status\": \"conflicts\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"message\": \"Conflicts detected. Resolve before retrying.\",\n }\n\n db.commit()\n\n return {\n \"status\": \"completed\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"total\": len(corrections),\n \"applied\": sum(1 for r in results if r[\"action\"] in (\"created\", \"updated\")),\n }\n # #endregion DictionaryManager.submit_bulk_corrections\n\n\n# #endregion DictionaryManager\n" + "body": "# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import]\n# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering.\n# @PRE Database session is open and valid.\n# @POST Dictionary and entry mutations are persisted with conflict detection.\n# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.\n# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary]\n# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary.\nclass DictionaryManager:\n # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]\n # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def create_dictionary(\n db: Session, name: str, source_dialect: str, target_dialect: str,\n created_by: Optional[str] = None, description: Optional[str] = None,\n is_active: bool = True,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.create_dictionary\"):\n log.reason(\"Creating dictionary\", payload={\"name\": name, \"source\": source_dialect, \"target\": target_dialect})\n dictionary = TerminologyDictionary(\n name=name,\n description=description,\n source_dialect=source_dialect,\n target_dialect=target_dialect,\n is_active=is_active,\n created_by=created_by,\n )\n db.add(dictionary)\n db.commit()\n db.refresh(dictionary)\n log.reflect(\"Dictionary created\", payload={\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.create_dictionary\n\n # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]\n # @BRIEF Update an existing terminology dictionary's metadata fields.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def update_dictionary(\n db: Session, dict_id: str, name: Optional[str] = None,\n description: Optional[str] = None, source_dialect: Optional[str] = None,\n target_dialect: Optional[str] = None, is_active: Optional[bool] = None,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.update_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n log.reason(\"Updating dictionary\", payload={\"id\": dict_id})\n if name is not None:\n dictionary.name = name\n if description is not None:\n dictionary.description = description\n if source_dialect is not None:\n dictionary.source_dialect = source_dialect\n if target_dialect is not None:\n dictionary.target_dialect = target_dialect\n if is_active is not None:\n dictionary.is_active = is_active\n db.commit()\n db.refresh(dictionary)\n log.reflect(\"Dictionary updated\", payload={\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.update_dictionary\n\n # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]\n # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).\n # @PRE dict_id exists.\n # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.\n # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.\n @staticmethod\n def delete_dictionary(db: Session, dict_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Check for attached active/scheduled jobs\n blocked_statuses = [\"ACTIVE\", \"READY\", \"RUNNING\", \"SCHEDULED\"]\n attached_jobs = (\n db.query(TranslationJobDictionary)\n .filter(\n TranslationJobDictionary.dictionary_id == dict_id,\n TranslationJobDictionary.job_id.in_(\n db.query(TranslationJob.id).filter(\n TranslationJob.status.in_(blocked_statuses)\n )\n ),\n )\n .count()\n )\n\n if attached_jobs > 0:\n log.explore(\"Delete blocked: dictionary attached to active jobs\", payload={\"dict_id\": dict_id, \"jobs\": attached_jobs}, error=f\"Dictionary attached to {attached_jobs} active job(s)\")\n raise ValueError(\n f\"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). \"\n \"Remove dictionary associations from jobs first.\"\n )\n\n log.reason(\"Deleting dictionary\", payload={\"id\": dict_id})\n # Delete entries and job-dictionary links first\n db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()\n db.delete(dictionary)\n db.commit()\n log.reflect(\"Dictionary deleted\", payload={\"id\": dict_id})\n # #endregion DictionaryManager.delete_dictionary\n\n # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get]\n # @BRIEF Get a single dictionary by ID with entry count.\n @staticmethod\n def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n return dictionary\n # #endregion DictionaryManager.get_dictionary\n\n # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list]\n # @BRIEF List dictionaries with pagination and total count.\n @staticmethod\n def list_dictionaries(\n db: Session, page: int = 1, page_size: int = 20,\n ) -> Tuple[List[TerminologyDictionary], int]:\n total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0\n dictionaries = (\n db.query(TerminologyDictionary)\n .order_by(TerminologyDictionary.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return dictionaries, total\n # #endregion DictionaryManager.list_dictionaries\n\n # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]\n # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def add_entry(\n db: Session, dict_id: str, source_term: str, target_term: str,\n context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.add_entry\"):\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}). Use overwrite or keep_existing conflict mode.\"\n )\n\n log.reason(\"Adding dictionary entry\", payload={\"dict_id\": dict_id, \"term\": source_term})\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=target_term.strip(),\n context_notes=context_notes.strip() if context_notes else None,\n )\n db.add(entry)\n db.commit()\n db.refresh(entry)\n log.reflect(\"Entry added\", payload={\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.add_entry\n\n # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]\n # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def edit_entry(\n db: Session, entry_id: str, source_term: Optional[str] = None,\n target_term: Optional[str] = None, context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.edit_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n\n log.reason(\"Editing dictionary entry\", payload={\"entry_id\": entry_id})\n if source_term is not None:\n normalized = _normalize_term(source_term)\n # Check uniqueness within the same dictionary\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == entry.dictionary_id,\n DictionaryEntry.source_term_normalized == normalized,\n DictionaryEntry.id != entry_id,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}).\"\n )\n entry.source_term = source_term.strip()\n entry.source_term_normalized = normalized\n if target_term is not None:\n entry.target_term = target_term.strip()\n if context_notes is not None:\n entry.context_notes = context_notes.strip() if context_notes else None\n db.commit()\n db.refresh(entry)\n log.reflect(\"Entry updated\", payload={\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.edit_entry\n\n # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]\n # @BRIEF Delete a single dictionary entry.\n @staticmethod\n def delete_entry(db: Session, entry_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n log.reason(\"Deleting dictionary entry\", payload={\"entry_id\": entry_id})\n db.delete(entry)\n db.commit()\n log.reflect(\"Entry deleted\", payload={\"entry_id\": entry_id})\n # #endregion DictionaryManager.delete_entry\n\n # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]\n # @BRIEF Delete all entries for a dictionary.\n @staticmethod\n def clear_entries(db: Session, dict_id: str) -> int:\n with belief_scope(\"DictionaryManager.clear_entries\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n log.reason(\"Clearing all entries\", payload={\"dict_id\": dict_id})\n deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.commit()\n log.reflect(\"Entries cleared\", payload={\"dict_id\": dict_id, \"count\": deleted})\n return deleted\n # #endregion DictionaryManager.clear_entries\n\n # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list]\n # @BRIEF List entries for a dictionary with pagination.\n @staticmethod\n def list_entries(\n db: Session, dict_id: str, page: int = 1, page_size: int = 50,\n ) -> Tuple[List[DictionaryEntry], int]:\n total = (\n db.query(func.count(DictionaryEntry.id))\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .scalar()\n or 0\n )\n entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id == dict_id)\n .order_by(DictionaryEntry.source_term.asc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n return entries, total\n # #endregion DictionaryManager.list_entries\n\n # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]\n # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).\n # @PRE content is valid CSV or TSV. dict_id exists.\n # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.\n # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.\n @staticmethod\n def import_entries(\n db: Session, dict_id: str, content: str,\n delimiter: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n preview_only: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.import_entries\"):\n # Validate dictionary\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Detect delimiter if not specified\n if not delimiter:\n delimiter = _detect_delimiter(content)\n log.reason(\"Detected delimiter\", payload={\"delimiter\": repr(delimiter)})\n\n if delimiter not in (\",\", \"\\t\"):\n raise ValueError(f\"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\\\t'.\")\n\n # Parse content\n reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)\n required_fields = {\"source_term\", \"target_term\"}\n if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):\n raise ValueError(\n f\"CSV/TSV must have at least 'source_term' and 'target_term' columns. \"\n f\"Got: {reader.fieldnames}\"\n )\n\n result: Dict[str, Any] = {\n \"total\": 0,\n \"created\": 0,\n \"updated\": 0,\n \"skipped\": 0,\n \"errors\": [],\n \"preview\": [],\n }\n\n rows = list(reader)\n result[\"total\"] = len(rows)\n\n for row_idx, row in enumerate(rows):\n try:\n source_term = row.get(\"source_term\", \"\").strip()\n target_term = row.get(\"target_term\", \"\").strip()\n context_notes = row.get(\"context_notes\", \"\").strip() or None\n\n if not source_term or not target_term:\n result[\"errors\"].append({\n \"line\": row_idx + 2, # +2 for header and 1-indexed\n \"error\": \"Both source_term and target_term are required\",\n \"row\": row,\n })\n continue\n\n normalized = _normalize_term(source_term)\n\n if preview_only:\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n preview_row = {\n \"line\": row_idx + 2,\n \"source_term\": source_term,\n \"target_term\": target_term,\n \"context_notes\": context_notes,\n \"is_conflict\": existing is not None,\n \"existing_target_term\": existing.target_term if existing else None,\n }\n result[\"preview\"].append(preview_row)\n continue\n\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n if on_conflict == \"overwrite\":\n existing.source_term = source_term\n existing.target_term = target_term\n existing.context_notes = context_notes\n result[\"updated\"] += 1\n elif on_conflict == \"keep_existing\":\n result[\"skipped\"] += 1\n else: # cancel\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": f\"Conflict on '{source_term}' and on_conflict='cancel'\",\n \"row\": row,\n })\n continue\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=target_term,\n context_notes=context_notes,\n )\n db.add(entry)\n result[\"created\"] += 1\n\n except Exception as e:\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": str(e),\n \"row\": row,\n })\n\n if not preview_only:\n db.commit()\n\n log.reflect(\"Import complete\", payload={\n \"dict_id\": dict_id,\n \"total\": result[\"total\"],\n \"created\": result[\"created\"],\n \"updated\": result[\"updated\"],\n \"skipped\": result[\"skipped\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n # #endregion DictionaryManager.import_entries\n\n # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]\n # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.\n # @PRE job_id exists and source_texts is a list of strings.\n # @POST Returns list of matched entries with match info, sorted by dictionary link priority.\n # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.\n @staticmethod\n def filter_for_batch(\n db: Session, source_texts: List[str], job_id: str,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"DictionaryManager.filter_for_batch\"):\n # Get dictionaries attached to this job\n job_dict_links = (\n db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n if not job_dict_links:\n log.reason(\"No dictionaries attached to job\", payload={\"job_id\": job_id})\n return []\n\n dict_ids = [jd.dictionary_id for jd in job_dict_links]\n\n # Get all active dictionaries\n dictionaries = (\n db.query(TerminologyDictionary)\n .filter(\n TerminologyDictionary.id.in_(dict_ids),\n TerminologyDictionary.is_active == True,\n )\n .all()\n )\n if not dictionaries:\n return []\n\n # Build dict_id -> dict_name lookup\n dict_map = {d.id: d.name for d in dictionaries}\n\n # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)\n all_entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id.in_(dict_ids))\n .order_by(DictionaryEntry.source_term.asc())\n .all()\n )\n\n if not all_entries:\n return []\n\n matched: List[Dict[str, Any]] = []\n seen_source_match: set = set()\n\n for text_idx, source_text in enumerate(source_texts):\n if not source_text:\n continue\n\n text_lower = source_text.lower()\n\n for entry in all_entries:\n norm = entry.source_term_normalized\n if not norm:\n continue\n\n # Word-boundary-aware matching\n # Build pattern: \\bterm\\b (case-insensitive)\n # Escape regex special chars in the search term\n escaped = re.escape(norm)\n pattern = re.compile(r\"\\b\" + escaped + r\"\\b\", re.IGNORECASE)\n\n if pattern.search(text_lower):\n match_key = (text_idx, entry.id)\n if match_key not in seen_source_match:\n seen_source_match.add(match_key)\n matched.append({\n \"text_index\": text_idx,\n \"source_text\": source_text,\n \"entry_id\": entry.id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"dictionary_id\": entry.dictionary_id,\n \"dictionary_name\": dict_map.get(entry.dictionary_id, \"\"),\n \"context_notes\": entry.context_notes,\n })\n\n # Sort by dictionary link priority (order of dict_ids from link order)\n dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}\n matched.sort(key=lambda m: (dict_priority.get(m[\"dictionary_id\"], 999), m[\"text_index\"]))\n\n log.reflect(\"Batch filter match complete\", payload={\n \"job_id\": job_id,\n \"source_texts\": len(source_texts),\n \"matches\": len(matched),\n \"dictionaries_used\": len(dictionaries),\n })\n return matched\n # #endregion DictionaryManager.filter_for_batch\n\n\n # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]\n # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.\n # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.\n # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.\n # @SIDE_EFFECT Creates/updates DictionaryEntry row.\n @staticmethod\n def submit_correction(\n db: Session,\n dict_id: str,\n source_term: str,\n incorrect_target_term: str,\n corrected_target_term: str,\n origin_run_id: Optional[str] = None,\n origin_row_key: Optional[str] = None,\n origin_user_id: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_correction\"):\n # Validate dictionary exists\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n result: Dict[str, Any] = {\n \"source_term\": source_term,\n \"target_term\": corrected_target_term,\n \"action\": \"created\",\n \"entry_id\": None,\n \"conflict\": None,\n \"message\": None,\n }\n\n if existing:\n # Conflict detected\n if on_conflict == \"keep_existing\":\n result[\"action\"] = \"conflict_detected\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"keep_existing\",\n }\n result[\"message\"] = f\"Existing entry kept: '{existing.target_term}'\"\n return result\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target_term.strip()\n if origin_run_id:\n existing.origin_run_id = origin_run_id\n if origin_row_key:\n existing.origin_row_key = origin_row_key\n if origin_user_id:\n existing.origin_user_id = origin_user_id\n db.flush()\n result[\"action\"] = \"updated\"\n result[\"entry_id\"] = existing.id\n result[\"message\"] = f\"Entry updated from '{existing.target_term}' to '{corrected_target_term}'\"\n else: # cancel\n result[\"action\"] = \"skipped\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"cancel\",\n }\n result[\"message\"] = \"Correction cancelled by conflict mode\"\n return result\n else:\n # Create new entry\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=corrected_target_term.strip(),\n origin_run_id=origin_run_id,\n origin_row_key=origin_row_key,\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n db.flush()\n result[\"entry_id\"] = entry.id\n result[\"message\"] = f\"Entry created for '{source_term}' -> '{corrected_target_term}'\"\n\n db.commit()\n log.reflect(\"Correction processed\", payload=result)\n return result\n # #endregion DictionaryManager.submit_correction\n\n # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk]\n # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict.\n # @PRE corrections list is non-empty. dict_id exists.\n # @POST All corrections applied or none applied with conflict list (atomic).\n # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.\n @staticmethod\n def submit_bulk_corrections(\n db: Session,\n dict_id: str,\n corrections: List[Dict[str, Any]],\n origin_user_id: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_bulk_corrections\"):\n # Validate dictionary first\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n results: List[Dict[str, Any]] = []\n conflicts: List[Dict[str, Any]] = []\n all_ok = True\n\n for corr in corrections:\n source_term = corr.get(\"source_term\", \"\").strip()\n incorrect_target = corr.get(\"incorrect_target_term\", \"\").strip()\n corrected_target = corr.get(\"corrected_target_term\", \"\").strip()\n\n if not source_term or not corrected_target:\n results.append({\n \"source_term\": source_term,\n \"action\": \"error\",\n \"message\": \"source_term and corrected_target_term are required\",\n })\n all_ok = False\n continue\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n on_conflict = corr.get(\"on_conflict\", \"overwrite\")\n if on_conflict == \"keep_existing\":\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"keep_existing\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"conflict_detected\",\n \"message\": f\"Existing entry kept: '{existing.target_term}'\",\n })\n continue\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target\n existing.origin_user_id = origin_user_id\n results.append({\n \"source_term\": source_term,\n \"action\": \"updated\",\n \"entry_id\": existing.id,\n \"message\": f\"Updated to '{corrected_target}'\",\n })\n else:\n conflicts.append({\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target,\n \"action\": \"cancel\",\n })\n results.append({\n \"source_term\": source_term,\n \"action\": \"skipped\",\n \"message\": \"Cancelled by conflict mode\",\n })\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=corrected_target,\n origin_run_id=corr.get(\"origin_run_id\"),\n origin_row_key=corr.get(\"origin_row_key\"),\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n results.append({\n \"source_term\": source_term,\n \"action\": \"created\",\n \"message\": f\"Entry created for '{source_term}' -> '{corrected_target}'\",\n })\n\n if conflicts and not all_ok:\n # Rollback — all failed\n db.rollback()\n return {\n \"status\": \"conflicts\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"message\": \"Conflicts detected. Resolve before retrying.\",\n }\n\n db.commit()\n\n return {\n \"status\": \"completed\",\n \"results\": results,\n \"conflicts\": conflicts,\n \"total\": len(corrections),\n \"applied\": sum(1 for r in results if r[\"action\"] in (\"created\", \"updated\")),\n }\n # #endregion DictionaryManager.submit_bulk_corrections\n\n\n# #endregion DictionaryManager\n" }, { "contract_id": "DictionaryManager.create_dictionary", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 38, - "end_line": 62, + "start_line": 41, + "end_line": 65, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -63735,14 +64744,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]\n # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def create_dictionary(\n db: Session, name: str, source_dialect: str, target_dialect: str,\n created_by: Optional[str] = None, description: Optional[str] = None,\n is_active: bool = True,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.create_dictionary\"):\n logger.reason(\"Creating dictionary\", {\"name\": name, \"source\": source_dialect, \"target\": target_dialect})\n dictionary = TerminologyDictionary(\n name=name,\n description=description,\n source_dialect=source_dialect,\n target_dialect=target_dialect,\n is_active=is_active,\n created_by=created_by,\n )\n db.add(dictionary)\n db.commit()\n db.refresh(dictionary)\n logger.reflect(\"Dictionary created\", {\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.create_dictionary\n" + "body": " # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create]\n # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def create_dictionary(\n db: Session, name: str, source_dialect: str, target_dialect: str,\n created_by: Optional[str] = None, description: Optional[str] = None,\n is_active: bool = True,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.create_dictionary\"):\n log.reason(\"Creating dictionary\", payload={\"name\": name, \"source\": source_dialect, \"target\": target_dialect})\n dictionary = TerminologyDictionary(\n name=name,\n description=description,\n source_dialect=source_dialect,\n target_dialect=target_dialect,\n is_active=is_active,\n created_by=created_by,\n )\n db.add(dictionary)\n db.commit()\n db.refresh(dictionary)\n log.reflect(\"Dictionary created\", payload={\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.create_dictionary\n" }, { "contract_id": "DictionaryManager.update_dictionary", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 64, - "end_line": 92, + "start_line": 67, + "end_line": 95, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -63775,14 +64784,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]\n # @BRIEF Update an existing terminology dictionary's metadata fields.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def update_dictionary(\n db: Session, dict_id: str, name: Optional[str] = None,\n description: Optional[str] = None, source_dialect: Optional[str] = None,\n target_dialect: Optional[str] = None, is_active: Optional[bool] = None,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.update_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n logger.reason(\"Updating dictionary\", {\"id\": dict_id})\n if name is not None:\n dictionary.name = name\n if description is not None:\n dictionary.description = description\n if source_dialect is not None:\n dictionary.source_dialect = source_dialect\n if target_dialect is not None:\n dictionary.target_dialect = target_dialect\n if is_active is not None:\n dictionary.is_active = is_active\n db.commit()\n db.refresh(dictionary)\n logger.reflect(\"Dictionary updated\", {\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.update_dictionary\n" + "body": " # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update]\n # @BRIEF Update an existing terminology dictionary's metadata fields.\n # @RELATION DEPENDS_ON -> [TerminologyDictionary]\n @staticmethod\n def update_dictionary(\n db: Session, dict_id: str, name: Optional[str] = None,\n description: Optional[str] = None, source_dialect: Optional[str] = None,\n target_dialect: Optional[str] = None, is_active: Optional[bool] = None,\n ) -> TerminologyDictionary:\n with belief_scope(\"DictionaryManager.update_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n log.reason(\"Updating dictionary\", payload={\"id\": dict_id})\n if name is not None:\n dictionary.name = name\n if description is not None:\n dictionary.description = description\n if source_dialect is not None:\n dictionary.source_dialect = source_dialect\n if target_dialect is not None:\n dictionary.target_dialect = target_dialect\n if is_active is not None:\n dictionary.is_active = is_active\n db.commit()\n db.refresh(dictionary)\n log.reflect(\"Dictionary updated\", payload={\"id\": dictionary.id})\n return dictionary\n # #endregion DictionaryManager.update_dictionary\n" }, { "contract_id": "DictionaryManager.delete_dictionary", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 94, - "end_line": 135, + "start_line": 97, + "end_line": 138, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -63820,14 +64829,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]\n # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).\n # @PRE dict_id exists.\n # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.\n # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.\n @staticmethod\n def delete_dictionary(db: Session, dict_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Check for attached active/scheduled jobs\n blocked_statuses = [\"ACTIVE\", \"READY\", \"RUNNING\", \"SCHEDULED\"]\n attached_jobs = (\n db.query(TranslationJobDictionary)\n .filter(\n TranslationJobDictionary.dictionary_id == dict_id,\n TranslationJobDictionary.job_id.in_(\n db.query(TranslationJob.id).filter(\n TranslationJob.status.in_(blocked_statuses)\n )\n ),\n )\n .count()\n )\n\n if attached_jobs > 0:\n logger.explore(\"Delete blocked: dictionary attached to active jobs\", {\"dict_id\": dict_id, \"jobs\": attached_jobs})\n raise ValueError(\n f\"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). \"\n \"Remove dictionary associations from jobs first.\"\n )\n\n logger.reason(\"Deleting dictionary\", {\"id\": dict_id})\n # Delete entries and job-dictionary links first\n db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()\n db.delete(dictionary)\n db.commit()\n logger.reflect(\"Dictionary deleted\", {\"id\": dict_id})\n # #endregion DictionaryManager.delete_dictionary\n" + "body": " # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete]\n # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard).\n # @PRE dict_id exists.\n # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.\n # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first.\n @staticmethod\n def delete_dictionary(db: Session, dict_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_dictionary\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Check for attached active/scheduled jobs\n blocked_statuses = [\"ACTIVE\", \"READY\", \"RUNNING\", \"SCHEDULED\"]\n attached_jobs = (\n db.query(TranslationJobDictionary)\n .filter(\n TranslationJobDictionary.dictionary_id == dict_id,\n TranslationJobDictionary.job_id.in_(\n db.query(TranslationJob.id).filter(\n TranslationJob.status.in_(blocked_statuses)\n )\n ),\n )\n .count()\n )\n\n if attached_jobs > 0:\n log.explore(\"Delete blocked: dictionary attached to active jobs\", payload={\"dict_id\": dict_id, \"jobs\": attached_jobs}, error=f\"Dictionary attached to {attached_jobs} active job(s)\")\n raise ValueError(\n f\"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). \"\n \"Remove dictionary associations from jobs first.\"\n )\n\n log.reason(\"Deleting dictionary\", payload={\"id\": dict_id})\n # Delete entries and job-dictionary links first\n db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete()\n db.delete(dictionary)\n db.commit()\n log.reflect(\"Dictionary deleted\", payload={\"id\": dict_id})\n # #endregion DictionaryManager.delete_dictionary\n" }, { "contract_id": "DictionaryManager.get_dictionary", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 137, - "end_line": 145, + "start_line": 140, + "end_line": 148, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -63859,8 +64868,8 @@ "contract_id": "DictionaryManager.list_dictionaries", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 147, - "end_line": 162, + "start_line": 150, + "end_line": 165, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -63892,8 +64901,8 @@ "contract_id": "DictionaryManager.add_entry", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 164, - "end_line": 201, + "start_line": 167, + "end_line": 204, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -63926,14 +64935,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]\n # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def add_entry(\n db: Session, dict_id: str, source_term: str, target_term: str,\n context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.add_entry\"):\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}). Use overwrite or keep_existing conflict mode.\"\n )\n\n logger.reason(\"Adding dictionary entry\", {\"dict_id\": dict_id, \"term\": source_term})\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=target_term.strip(),\n context_notes=context_notes.strip() if context_notes else None,\n )\n db.add(entry)\n db.commit()\n db.refresh(entry)\n logger.reflect(\"Entry added\", {\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.add_entry\n" + "body": " # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add]\n # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def add_entry(\n db: Session, dict_id: str, source_term: str, target_term: str,\n context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.add_entry\"):\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}). Use overwrite or keep_existing conflict mode.\"\n )\n\n log.reason(\"Adding dictionary entry\", payload={\"dict_id\": dict_id, \"term\": source_term})\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=target_term.strip(),\n context_notes=context_notes.strip() if context_notes else None,\n )\n db.add(entry)\n db.commit()\n db.refresh(entry)\n log.reflect(\"Entry added\", payload={\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.add_entry\n" }, { "contract_id": "DictionaryManager.edit_entry", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 203, - "end_line": 244, + "start_line": 206, + "end_line": 247, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -63966,14 +64975,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]\n # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def edit_entry(\n db: Session, entry_id: str, source_term: Optional[str] = None,\n target_term: Optional[str] = None, context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.edit_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n\n logger.reason(\"Editing dictionary entry\", {\"entry_id\": entry_id})\n if source_term is not None:\n normalized = _normalize_term(source_term)\n # Check uniqueness within the same dictionary\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == entry.dictionary_id,\n DictionaryEntry.source_term_normalized == normalized,\n DictionaryEntry.id != entry_id,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}).\"\n )\n entry.source_term = source_term.strip()\n entry.source_term_normalized = normalized\n if target_term is not None:\n entry.target_term = target_term.strip()\n if context_notes is not None:\n entry.context_notes = context_notes.strip() if context_notes else None\n db.commit()\n db.refresh(entry)\n logger.reflect(\"Entry updated\", {\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.edit_entry\n" + "body": " # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit]\n # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check.\n # @RELATION DEPENDS_ON -> [DictionaryEntry]\n @staticmethod\n def edit_entry(\n db: Session, entry_id: str, source_term: Optional[str] = None,\n target_term: Optional[str] = None, context_notes: Optional[str] = None,\n ) -> DictionaryEntry:\n with belief_scope(\"DictionaryManager.edit_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n\n log.reason(\"Editing dictionary entry\", payload={\"entry_id\": entry_id})\n if source_term is not None:\n normalized = _normalize_term(source_term)\n # Check uniqueness within the same dictionary\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == entry.dictionary_id,\n DictionaryEntry.source_term_normalized == normalized,\n DictionaryEntry.id != entry_id,\n )\n .first()\n )\n if existing:\n raise ValueError(\n f\"Duplicate entry: '{source_term}' already exists in this dictionary \"\n f\"(id={existing.id}).\"\n )\n entry.source_term = source_term.strip()\n entry.source_term_normalized = normalized\n if target_term is not None:\n entry.target_term = target_term.strip()\n if context_notes is not None:\n entry.context_notes = context_notes.strip() if context_notes else None\n db.commit()\n db.refresh(entry)\n log.reflect(\"Entry updated\", payload={\"entry_id\": entry.id})\n return entry\n # #endregion DictionaryManager.edit_entry\n" }, { "contract_id": "DictionaryManager.delete_entry", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 246, - "end_line": 258, + "start_line": 249, + "end_line": 261, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -63999,14 +65008,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]\n # @BRIEF Delete a single dictionary entry.\n @staticmethod\n def delete_entry(db: Session, entry_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n logger.reason(\"Deleting dictionary entry\", {\"entry_id\": entry_id})\n db.delete(entry)\n db.commit()\n logger.reflect(\"Entry deleted\", {\"entry_id\": entry_id})\n # #endregion DictionaryManager.delete_entry\n" + "body": " # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete]\n # @BRIEF Delete a single dictionary entry.\n @staticmethod\n def delete_entry(db: Session, entry_id: str) -> None:\n with belief_scope(\"DictionaryManager.delete_entry\"):\n entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()\n if not entry:\n raise ValueError(f\"Entry not found: {entry_id}\")\n log.reason(\"Deleting dictionary entry\", payload={\"entry_id\": entry_id})\n db.delete(entry)\n db.commit()\n log.reflect(\"Entry deleted\", payload={\"entry_id\": entry_id})\n # #endregion DictionaryManager.delete_entry\n" }, { "contract_id": "DictionaryManager.clear_entries", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 260, - "end_line": 273, + "start_line": 263, + "end_line": 276, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -64032,14 +65041,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]\n # @BRIEF Delete all entries for a dictionary.\n @staticmethod\n def clear_entries(db: Session, dict_id: str) -> int:\n with belief_scope(\"DictionaryManager.clear_entries\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n logger.reason(\"Clearing all entries\", {\"dict_id\": dict_id})\n deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.commit()\n logger.reflect(\"Entries cleared\", {\"dict_id\": dict_id, \"count\": deleted})\n return deleted\n # #endregion DictionaryManager.clear_entries\n" + "body": " # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear]\n # @BRIEF Delete all entries for a dictionary.\n @staticmethod\n def clear_entries(db: Session, dict_id: str) -> int:\n with belief_scope(\"DictionaryManager.clear_entries\"):\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n log.reason(\"Clearing all entries\", payload={\"dict_id\": dict_id})\n deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete()\n db.commit()\n log.reflect(\"Entries cleared\", payload={\"dict_id\": dict_id, \"count\": deleted})\n return deleted\n # #endregion DictionaryManager.clear_entries\n" }, { "contract_id": "DictionaryManager.list_entries", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 275, - "end_line": 296, + "start_line": 278, + "end_line": 299, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -64071,8 +65080,8 @@ "contract_id": "DictionaryManager.import_entries", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 298, - "end_line": 435, + "start_line": 301, + "end_line": 438, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64110,14 +65119,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]\n # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).\n # @PRE content is valid CSV or TSV. dict_id exists.\n # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.\n # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.\n @staticmethod\n def import_entries(\n db: Session, dict_id: str, content: str,\n delimiter: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n preview_only: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.import_entries\"):\n # Validate dictionary\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Detect delimiter if not specified\n if not delimiter:\n delimiter = _detect_delimiter(content)\n logger.reason(\"Detected delimiter\", {\"delimiter\": repr(delimiter)})\n\n if delimiter not in (\",\", \"\\t\"):\n raise ValueError(f\"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\\\t'.\")\n\n # Parse content\n reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)\n required_fields = {\"source_term\", \"target_term\"}\n if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):\n raise ValueError(\n f\"CSV/TSV must have at least 'source_term' and 'target_term' columns. \"\n f\"Got: {reader.fieldnames}\"\n )\n\n result: Dict[str, Any] = {\n \"total\": 0,\n \"created\": 0,\n \"updated\": 0,\n \"skipped\": 0,\n \"errors\": [],\n \"preview\": [],\n }\n\n rows = list(reader)\n result[\"total\"] = len(rows)\n\n for row_idx, row in enumerate(rows):\n try:\n source_term = row.get(\"source_term\", \"\").strip()\n target_term = row.get(\"target_term\", \"\").strip()\n context_notes = row.get(\"context_notes\", \"\").strip() or None\n\n if not source_term or not target_term:\n result[\"errors\"].append({\n \"line\": row_idx + 2, # +2 for header and 1-indexed\n \"error\": \"Both source_term and target_term are required\",\n \"row\": row,\n })\n continue\n\n normalized = _normalize_term(source_term)\n\n if preview_only:\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n preview_row = {\n \"line\": row_idx + 2,\n \"source_term\": source_term,\n \"target_term\": target_term,\n \"context_notes\": context_notes,\n \"is_conflict\": existing is not None,\n \"existing_target_term\": existing.target_term if existing else None,\n }\n result[\"preview\"].append(preview_row)\n continue\n\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n if on_conflict == \"overwrite\":\n existing.source_term = source_term\n existing.target_term = target_term\n existing.context_notes = context_notes\n result[\"updated\"] += 1\n elif on_conflict == \"keep_existing\":\n result[\"skipped\"] += 1\n else: # cancel\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": f\"Conflict on '{source_term}' and on_conflict='cancel'\",\n \"row\": row,\n })\n continue\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=target_term,\n context_notes=context_notes,\n )\n db.add(entry)\n result[\"created\"] += 1\n\n except Exception as e:\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": str(e),\n \"row\": row,\n })\n\n if not preview_only:\n db.commit()\n\n logger.reflect(\"Import complete\", {\n \"dict_id\": dict_id,\n \"total\": result[\"total\"],\n \"created\": result[\"created\"],\n \"updated\": result[\"updated\"],\n \"skipped\": result[\"skipped\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n # #endregion DictionaryManager.import_entries\n" + "body": " # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv]\n # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel).\n # @PRE content is valid CSV or TSV. dict_id exists.\n # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support.\n # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.\n @staticmethod\n def import_entries(\n db: Session, dict_id: str, content: str,\n delimiter: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n preview_only: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.import_entries\"):\n # Validate dictionary\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary not found: {dict_id}\")\n\n # Detect delimiter if not specified\n if not delimiter:\n delimiter = _detect_delimiter(content)\n log.reason(\"Detected delimiter\", payload={\"delimiter\": repr(delimiter)})\n\n if delimiter not in (\",\", \"\\t\"):\n raise ValueError(f\"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\\\t'.\")\n\n # Parse content\n reader = csv.DictReader(io.StringIO(content), delimiter=delimiter)\n required_fields = {\"source_term\", \"target_term\"}\n if not reader.fieldnames or not required_fields.issubset(reader.fieldnames):\n raise ValueError(\n f\"CSV/TSV must have at least 'source_term' and 'target_term' columns. \"\n f\"Got: {reader.fieldnames}\"\n )\n\n result: Dict[str, Any] = {\n \"total\": 0,\n \"created\": 0,\n \"updated\": 0,\n \"skipped\": 0,\n \"errors\": [],\n \"preview\": [],\n }\n\n rows = list(reader)\n result[\"total\"] = len(rows)\n\n for row_idx, row in enumerate(rows):\n try:\n source_term = row.get(\"source_term\", \"\").strip()\n target_term = row.get(\"target_term\", \"\").strip()\n context_notes = row.get(\"context_notes\", \"\").strip() or None\n\n if not source_term or not target_term:\n result[\"errors\"].append({\n \"line\": row_idx + 2, # +2 for header and 1-indexed\n \"error\": \"Both source_term and target_term are required\",\n \"row\": row,\n })\n continue\n\n normalized = _normalize_term(source_term)\n\n if preview_only:\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n preview_row = {\n \"line\": row_idx + 2,\n \"source_term\": source_term,\n \"target_term\": target_term,\n \"context_notes\": context_notes,\n \"is_conflict\": existing is not None,\n \"existing_target_term\": existing.target_term if existing else None,\n }\n result[\"preview\"].append(preview_row)\n continue\n\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n if existing:\n if on_conflict == \"overwrite\":\n existing.source_term = source_term\n existing.target_term = target_term\n existing.context_notes = context_notes\n result[\"updated\"] += 1\n elif on_conflict == \"keep_existing\":\n result[\"skipped\"] += 1\n else: # cancel\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": f\"Conflict on '{source_term}' and on_conflict='cancel'\",\n \"row\": row,\n })\n continue\n else:\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term,\n source_term_normalized=normalized,\n target_term=target_term,\n context_notes=context_notes,\n )\n db.add(entry)\n result[\"created\"] += 1\n\n except Exception as e:\n result[\"errors\"].append({\n \"line\": row_idx + 2,\n \"error\": str(e),\n \"row\": row,\n })\n\n if not preview_only:\n db.commit()\n\n log.reflect(\"Import complete\", payload={\n \"dict_id\": dict_id,\n \"total\": result[\"total\"],\n \"created\": result[\"created\"],\n \"updated\": result[\"updated\"],\n \"skipped\": result[\"skipped\"],\n \"errors\": len(result[\"errors\"]),\n })\n return result\n # #endregion DictionaryManager.import_entries\n" }, { "contract_id": "DictionaryManager.filter_for_batch", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 437, - "end_line": 532, + "start_line": 440, + "end_line": 535, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64155,14 +65164,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]\n # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.\n # @PRE job_id exists and source_texts is a list of strings.\n # @POST Returns list of matched entries with match info, sorted by dictionary link priority.\n # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.\n @staticmethod\n def filter_for_batch(\n db: Session, source_texts: List[str], job_id: str,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"DictionaryManager.filter_for_batch\"):\n # Get dictionaries attached to this job\n job_dict_links = (\n db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n if not job_dict_links:\n logger.reason(\"No dictionaries attached to job\", {\"job_id\": job_id})\n return []\n\n dict_ids = [jd.dictionary_id for jd in job_dict_links]\n\n # Get all active dictionaries\n dictionaries = (\n db.query(TerminologyDictionary)\n .filter(\n TerminologyDictionary.id.in_(dict_ids),\n TerminologyDictionary.is_active == True,\n )\n .all()\n )\n if not dictionaries:\n return []\n\n # Build dict_id -> dict_name lookup\n dict_map = {d.id: d.name for d in dictionaries}\n\n # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)\n all_entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id.in_(dict_ids))\n .order_by(DictionaryEntry.source_term.asc())\n .all()\n )\n\n if not all_entries:\n return []\n\n matched: List[Dict[str, Any]] = []\n seen_source_match: set = set()\n\n for text_idx, source_text in enumerate(source_texts):\n if not source_text:\n continue\n\n text_lower = source_text.lower()\n\n for entry in all_entries:\n norm = entry.source_term_normalized\n if not norm:\n continue\n\n # Word-boundary-aware matching\n # Build pattern: \\bterm\\b (case-insensitive)\n # Escape regex special chars in the search term\n escaped = re.escape(norm)\n pattern = re.compile(r\"\\b\" + escaped + r\"\\b\", re.IGNORECASE)\n\n if pattern.search(text_lower):\n match_key = (text_idx, entry.id)\n if match_key not in seen_source_match:\n seen_source_match.add(match_key)\n matched.append({\n \"text_index\": text_idx,\n \"source_text\": source_text,\n \"entry_id\": entry.id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"dictionary_id\": entry.dictionary_id,\n \"dictionary_name\": dict_map.get(entry.dictionary_id, \"\"),\n \"context_notes\": entry.context_notes,\n })\n\n # Sort by dictionary link priority (order of dict_ids from link order)\n dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}\n matched.sort(key=lambda m: (dict_priority.get(m[\"dictionary_id\"], 999), m[\"text_index\"]))\n\n logger.reflect(\"Batch filter match complete\", {\n \"job_id\": job_id,\n \"source_texts\": len(source_texts),\n \"matches\": len(matched),\n \"dictionaries_used\": len(dictionaries),\n })\n return matched\n # #endregion DictionaryManager.filter_for_batch\n" + "body": " # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch]\n # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.\n # @PRE job_id exists and source_texts is a list of strings.\n # @POST Returns list of matched entries with match info, sorted by dictionary link priority.\n # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.\n @staticmethod\n def filter_for_batch(\n db: Session, source_texts: List[str], job_id: str,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"DictionaryManager.filter_for_batch\"):\n # Get dictionaries attached to this job\n job_dict_links = (\n db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n if not job_dict_links:\n log.reason(\"No dictionaries attached to job\", payload={\"job_id\": job_id})\n return []\n\n dict_ids = [jd.dictionary_id for jd in job_dict_links]\n\n # Get all active dictionaries\n dictionaries = (\n db.query(TerminologyDictionary)\n .filter(\n TerminologyDictionary.id.in_(dict_ids),\n TerminologyDictionary.is_active == True,\n )\n .all()\n )\n if not dictionaries:\n return []\n\n # Build dict_id -> dict_name lookup\n dict_map = {d.id: d.name for d in dictionaries}\n\n # Fetch all entries for these dictionaries, ordered by dictionary (priority order from link order)\n all_entries = (\n db.query(DictionaryEntry)\n .filter(DictionaryEntry.dictionary_id.in_(dict_ids))\n .order_by(DictionaryEntry.source_term.asc())\n .all()\n )\n\n if not all_entries:\n return []\n\n matched: List[Dict[str, Any]] = []\n seen_source_match: set = set()\n\n for text_idx, source_text in enumerate(source_texts):\n if not source_text:\n continue\n\n text_lower = source_text.lower()\n\n for entry in all_entries:\n norm = entry.source_term_normalized\n if not norm:\n continue\n\n # Word-boundary-aware matching\n # Build pattern: \\bterm\\b (case-insensitive)\n # Escape regex special chars in the search term\n escaped = re.escape(norm)\n pattern = re.compile(r\"\\b\" + escaped + r\"\\b\", re.IGNORECASE)\n\n if pattern.search(text_lower):\n match_key = (text_idx, entry.id)\n if match_key not in seen_source_match:\n seen_source_match.add(match_key)\n matched.append({\n \"text_index\": text_idx,\n \"source_text\": source_text,\n \"entry_id\": entry.id,\n \"source_term\": entry.source_term,\n \"source_term_normalized\": entry.source_term_normalized,\n \"target_term\": entry.target_term,\n \"dictionary_id\": entry.dictionary_id,\n \"dictionary_name\": dict_map.get(entry.dictionary_id, \"\"),\n \"context_notes\": entry.context_notes,\n })\n\n # Sort by dictionary link priority (order of dict_ids from link order)\n dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)}\n matched.sort(key=lambda m: (dict_priority.get(m[\"dictionary_id\"], 999), m[\"text_index\"]))\n\n log.reflect(\"Batch filter match complete\", payload={\n \"job_id\": job_id,\n \"source_texts\": len(source_texts),\n \"matches\": len(matched),\n \"dictionaries_used\": len(dictionaries),\n })\n return matched\n # #endregion DictionaryManager.filter_for_batch\n" }, { "contract_id": "DictionaryManager.submit_correction", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 535, - "end_line": 630, + "start_line": 538, + "end_line": 633, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64200,14 +65209,14 @@ } ], "anchor_syntax": "region", - "body": " # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]\n # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.\n # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.\n # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.\n # @SIDE_EFFECT Creates/updates DictionaryEntry row.\n @staticmethod\n def submit_correction(\n db: Session,\n dict_id: str,\n source_term: str,\n incorrect_target_term: str,\n corrected_target_term: str,\n origin_run_id: Optional[str] = None,\n origin_row_key: Optional[str] = None,\n origin_user_id: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_correction\"):\n # Validate dictionary exists\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n result: Dict[str, Any] = {\n \"source_term\": source_term,\n \"target_term\": corrected_target_term,\n \"action\": \"created\",\n \"entry_id\": None,\n \"conflict\": None,\n \"message\": None,\n }\n\n if existing:\n # Conflict detected\n if on_conflict == \"keep_existing\":\n result[\"action\"] = \"conflict_detected\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"keep_existing\",\n }\n result[\"message\"] = f\"Existing entry kept: '{existing.target_term}'\"\n return result\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target_term.strip()\n if origin_run_id:\n existing.origin_run_id = origin_run_id\n if origin_row_key:\n existing.origin_row_key = origin_row_key\n if origin_user_id:\n existing.origin_user_id = origin_user_id\n db.flush()\n result[\"action\"] = \"updated\"\n result[\"entry_id\"] = existing.id\n result[\"message\"] = f\"Entry updated from '{existing.target_term}' to '{corrected_target_term}'\"\n else: # cancel\n result[\"action\"] = \"skipped\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"cancel\",\n }\n result[\"message\"] = \"Correction cancelled by conflict mode\"\n return result\n else:\n # Create new entry\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=corrected_target_term.strip(),\n origin_run_id=origin_run_id,\n origin_row_key=origin_row_key,\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n db.flush()\n result[\"entry_id\"] = entry.id\n result[\"message\"] = f\"Entry created for '{source_term}' -> '{corrected_target_term}'\"\n\n db.commit()\n logger.reflect(\"Correction processed\", result)\n return result\n # #endregion DictionaryManager.submit_correction\n" + "body": " # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit]\n # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking.\n # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.\n # @POST Entry created or updated; origin tracking populated. Returns action + conflict info.\n # @SIDE_EFFECT Creates/updates DictionaryEntry row.\n @staticmethod\n def submit_correction(\n db: Session,\n dict_id: str,\n source_term: str,\n incorrect_target_term: str,\n corrected_target_term: str,\n origin_run_id: Optional[str] = None,\n origin_row_key: Optional[str] = None,\n origin_user_id: Optional[str] = None,\n on_conflict: str = \"overwrite\",\n ) -> Dict[str, Any]:\n with belief_scope(\"DictionaryManager.submit_correction\"):\n # Validate dictionary exists\n dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()\n if not dictionary:\n raise ValueError(f\"Dictionary '{dict_id}' not found\")\n\n normalized = _normalize_term(source_term)\n existing = (\n db.query(DictionaryEntry)\n .filter(\n DictionaryEntry.dictionary_id == dict_id,\n DictionaryEntry.source_term_normalized == normalized,\n )\n .first()\n )\n\n result: Dict[str, Any] = {\n \"source_term\": source_term,\n \"target_term\": corrected_target_term,\n \"action\": \"created\",\n \"entry_id\": None,\n \"conflict\": None,\n \"message\": None,\n }\n\n if existing:\n # Conflict detected\n if on_conflict == \"keep_existing\":\n result[\"action\"] = \"conflict_detected\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"keep_existing\",\n }\n result[\"message\"] = f\"Existing entry kept: '{existing.target_term}'\"\n return result\n elif on_conflict == \"overwrite\":\n existing.target_term = corrected_target_term.strip()\n if origin_run_id:\n existing.origin_run_id = origin_run_id\n if origin_row_key:\n existing.origin_row_key = origin_row_key\n if origin_user_id:\n existing.origin_user_id = origin_user_id\n db.flush()\n result[\"action\"] = \"updated\"\n result[\"entry_id\"] = existing.id\n result[\"message\"] = f\"Entry updated from '{existing.target_term}' to '{corrected_target_term}'\"\n else: # cancel\n result[\"action\"] = \"skipped\"\n result[\"conflict\"] = {\n \"source_term\": source_term,\n \"existing_target_term\": existing.target_term,\n \"submitted_target_term\": corrected_target_term,\n \"action\": \"cancel\",\n }\n result[\"message\"] = \"Correction cancelled by conflict mode\"\n return result\n else:\n # Create new entry\n entry = DictionaryEntry(\n dictionary_id=dict_id,\n source_term=source_term.strip(),\n source_term_normalized=normalized,\n target_term=corrected_target_term.strip(),\n origin_run_id=origin_run_id,\n origin_row_key=origin_row_key,\n origin_user_id=origin_user_id,\n )\n db.add(entry)\n db.flush()\n result[\"entry_id\"] = entry.id\n result[\"message\"] = f\"Entry created for '{source_term}' -> '{corrected_target_term}'\"\n\n db.commit()\n log.reflect(\"Correction processed\", payload=result)\n return result\n # #endregion DictionaryManager.submit_correction\n" }, { "contract_id": "DictionaryManager.submit_bulk_corrections", "contract_type": "Function", "file_path": "backend/src/plugins/translate/dictionary.py", - "start_line": 632, - "end_line": 750, + "start_line": 635, + "end_line": 753, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64252,7 +65261,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/translate/events.py", "start_line": 1, - "end_line": 304, + "end_line": 307, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -64299,14 +65308,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging]\n# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationEvent]\n# @RELATION DEPENDS_ON -> [MetricSnapshot]\n# @PRE Database session is open and valid.\n# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.\n# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.\n# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]\n# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.\n# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.\n# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.\n# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\nfrom datetime import datetime, timezone, timedelta\nimport uuid\n\nfrom ...core.logger import logger, belief_scope\nfrom ...models.translate import TranslationEvent, MetricSnapshot\n\n# Terminal events per run — exactly one of these must exist for a completed run.\nTERMINAL_EVENT_TYPES = {\"RUN_COMPLETED\", \"RUN_FAILED\", \"RUN_CANCELLED\"}\n# All valid event types for validation.\nVALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {\n \"JOB_CREATED\", \"JOB_UPDATED\", \"JOB_DELETED\",\n \"RUN_STARTED\", \"RUN_RETRYING\", \"RUN_RETRY_INSERT\",\n \"BATCH_STARTED\", \"BATCH_COMPLETED\", \"BATCH_FAILED\", \"BATCH_RETRYING\",\n \"TRANSLATION_PHASE_STARTED\", \"TRANSLATION_PHASE_COMPLETED\",\n \"INSERT_PHASE_STARTED\", \"INSERT_PHASE_COMPLETED\",\n \"PREVIEW_CREATED\", \"PREVIEW_ACCEPTED\", \"PREVIEW_DISCARDED\",\n \"SCHEDULE_CREATED\", \"SCHEDULE_UPDATED\", \"SCHEDULE_DELETED\",\n \"METRICS_SNAPSHOT_CREATED\",\n}\n\nDEFAULT_RETENTION_DAYS = 90\n\n\n# #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants]\n# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning.\n# @PRE Database session is available.\n# @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot.\n# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.\n# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]\n# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.\nclass TranslationEventLog:\n\n def __init__(self, db: Session):\n self.db = db\n\n # #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create]\n # @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves.\n # @RELATION DEPENDS_ON -> [TranslationEvent]\n def _create_event_raw(\n self,\n job_id: str,\n event_type: str,\n payload: Optional[Dict[str, Any]] = None,\n run_id: Optional[str] = None,\n created_by: Optional[str] = None,\n ) -> TranslationEvent:\n event = TranslationEvent(\n id=str(uuid.uuid4()),\n job_id=job_id,\n run_id=run_id,\n event_type=event_type,\n event_data=payload or {},\n created_by=created_by,\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(event)\n self.db.flush()\n return event\n # #endregion _create_event_raw\n\n # #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log]\n # @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id.\n # @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants.\n # @POST TranslationEvent row is created; invariants checked before write.\n # @SIDE_EFFECT DB write.\n def log_event(\n self,\n job_id: str,\n event_type: str,\n payload: Optional[Dict[str, Any]] = None,\n run_id: Optional[str] = None,\n created_by: Optional[str] = None,\n ) -> TranslationEvent:\n with belief_scope(\"TranslationEventLog.log_event\"):\n if event_type not in VALID_EVENT_TYPES:\n raise ValueError(\n f\"Invalid event_type '{event_type}'. \"\n f\"Must be one of: {', '.join(sorted(VALID_EVENT_TYPES))}\"\n )\n\n # Enforce terminal event invariant for non-null run_id\n if run_id is not None and event_type in TERMINAL_EVENT_TYPES:\n existing_terminal = (\n self.db.query(TranslationEvent.id)\n .filter(\n TranslationEvent.run_id == run_id,\n TranslationEvent.event_type.in_(TERMINAL_EVENT_TYPES),\n )\n .first()\n )\n if existing_terminal:\n existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id)\n raise ValueError(\n f\"Run '{run_id}' already has a terminal event \"\n f\"({existing_id}). Cannot add another terminal event.\"\n )\n\n # Enforce run_started invariant — must exist before other run events\n if run_id is not None and event_type not in {\"RUN_STARTED\"} | TERMINAL_EVENT_TYPES:\n has_started = (\n self.db.query(TranslationEvent.id)\n .filter(\n TranslationEvent.run_id == run_id,\n TranslationEvent.event_type == \"RUN_STARTED\",\n )\n .first()\n )\n if not has_started:\n raise ValueError(\n f\"Cannot log '{event_type}' for run '{run_id}': \"\n f\"RUN_STARTED event must precede other run events.\"\n )\n\n event = self._create_event_raw(\n job_id=job_id,\n event_type=event_type,\n payload=payload,\n run_id=run_id,\n created_by=created_by,\n )\n logger.reason(f\"Event logged: {event_type}\", {\n \"event_id\": event.id,\n \"job_id\": job_id,\n \"run_id\": run_id,\n \"event_type\": event_type,\n })\n return event\n # #endregion log_event\n\n # #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query]\n # @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination.\n def query_events(\n self,\n job_id: Optional[str] = None,\n run_id: Optional[str] = None,\n event_type: Optional[str] = None,\n limit: int = 100,\n offset: int = 0,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationEventLog.query_events\"):\n query = self.db.query(TranslationEvent)\n\n if job_id:\n query = query.filter(TranslationEvent.job_id == job_id)\n if run_id:\n query = query.filter(TranslationEvent.run_id == run_id)\n if event_type:\n query = query.filter(TranslationEvent.event_type == event_type)\n\n events = (\n query.order_by(TranslationEvent.created_at.desc())\n .offset(offset)\n .limit(limit)\n .all()\n )\n\n return [\n {\n \"id\": e.id,\n \"job_id\": e.job_id,\n \"run_id\": e.run_id,\n \"event_type\": e.event_type,\n \"event_data\": e.event_data or {},\n \"created_by\": e.created_by,\n \"created_at\": e.created_at.isoformat() if e.created_at else None,\n }\n for e in events\n ]\n # #endregion query_events\n\n # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]\n # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.\n # @PRE None.\n # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.\n # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.\n def prune_expired(\n self,\n retention_days: int = DEFAULT_RETENTION_DAYS,\n batch_size: int = 1000,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.prune_expired\"):\n cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)\n logger.reason(\"Pruning expired events\", {\n \"cutoff\": cutoff.isoformat(),\n \"retention_days\": retention_days,\n })\n\n # Find events to prune\n expired_query = self.db.query(TranslationEvent).filter(\n TranslationEvent.created_at < cutoff\n )\n total_expired = expired_query.count()\n\n if total_expired == 0:\n logger.reflect(\"No expired events to prune\", {})\n return {\"pruned\": 0, \"snapshot_id\": None}\n\n # Create MetricSnapshot before pruning\n snapshot = MetricSnapshot(\n id=str(uuid.uuid4()),\n job_id=\"_prune_aggregate_\",\n key_hash=f\"prune_{cutoff.timestamp():.0f}\",\n covers_events_before=cutoff,\n total_records=total_expired,\n snapshot_date=datetime.now(timezone.utc),\n )\n self.db.add(snapshot)\n self.db.flush()\n snapshot_id = snapshot.id\n\n # Delete in batches\n pruned = 0\n while True:\n batch_ids = (\n self.db.query(TranslationEvent.id)\n .filter(TranslationEvent.created_at < cutoff)\n .limit(batch_size)\n .all()\n )\n if not batch_ids:\n break\n ids = [row[0] for row in batch_ids]\n self.db.query(TranslationEvent).filter(\n TranslationEvent.id.in_(ids)\n ).delete(synchronize_session=False)\n self.db.flush()\n pruned += len(ids)\n logger.reason(\"Pruned batch\", {\"batch_size\": len(ids), \"total_pruned\": pruned})\n\n self.db.commit()\n\n logger.reflect(\"Pruning complete\", {\n \"pruned\": pruned,\n \"snapshot_id\": snapshot_id,\n })\n return {\"pruned\": pruned, \"snapshot_id\": snapshot_id}\n # #endregion prune_expired\n\n # #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary]\n # @BRIEF Get a summary of events for a run, including invariant validity check.\n # @RELATION DEPENDS_ON -> [TranslationEvent]\n def get_run_event_summary(self, run_id: str) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.get_run_event_summary\"):\n events = self.query_events(run_id=run_id)\n\n has_started = any(e[\"event_type\"] == \"RUN_STARTED\" for e in events)\n terminal_events = [e for e in events if e[\"event_type\"] in TERMINAL_EVENT_TYPES]\n\n return {\n \"run_id\": run_id,\n \"event_count\": len(events),\n \"has_run_started\": has_started,\n \"terminal_event_count\": len(terminal_events),\n \"terminal_events\": terminal_events,\n \"invariant_valid\": has_started and len(terminal_events) <= 1,\n \"events\": events,\n }\n # #endregion get_run_event_summary\n\n # #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants]\n # @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance.\n # @RELATION DEPENDS_ON -> [TranslationEvent]\n def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.get_run_event_invariants_lightweight\"):\n # Lightweight query: only fetch event_type column, not event_data\n event_types = [\n row[0]\n for row in (\n self.db.query(TranslationEvent.event_type)\n .filter(TranslationEvent.run_id == run_id)\n .order_by(TranslationEvent.created_at.desc())\n .limit(100)\n .all()\n )\n ]\n\n has_run_started = \"RUN_STARTED\" in event_types\n terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES)\n\n return {\n \"has_run_started\": has_run_started,\n \"terminal_event_count\": terminal_count,\n \"invariant_valid\": has_run_started and terminal_count <= 1,\n }\n # #endregion get_run_event_invariants_lightweight\n\n\n# #endregion TranslationEventLog\n# #endregion TranslationEventLog\n" + "body": "# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging]\n# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationEvent]\n# @RELATION DEPENDS_ON -> [MetricSnapshot]\n# @PRE Database session is open and valid.\n# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run.\n# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion.\n# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]\n# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.\n# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events.\n# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.\n# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit.\nfrom typing import Any, Dict, List, Optional\nfrom sqlalchemy.orm import Session\nfrom datetime import datetime, timezone, timedelta\nimport uuid\n\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.logger import belief_scope\nfrom ...models.translate import TranslationEvent, MetricSnapshot\n\nlog = MarkerLogger(\"EventLog\")\n\n# Terminal events per run — exactly one of these must exist for a completed run.\nTERMINAL_EVENT_TYPES = {\"RUN_COMPLETED\", \"RUN_FAILED\", \"RUN_CANCELLED\"}\n# All valid event types for validation.\nVALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {\n \"JOB_CREATED\", \"JOB_UPDATED\", \"JOB_DELETED\",\n \"RUN_STARTED\", \"RUN_RETRYING\", \"RUN_RETRY_INSERT\",\n \"BATCH_STARTED\", \"BATCH_COMPLETED\", \"BATCH_FAILED\", \"BATCH_RETRYING\",\n \"TRANSLATION_PHASE_STARTED\", \"TRANSLATION_PHASE_COMPLETED\",\n \"INSERT_PHASE_STARTED\", \"INSERT_PHASE_COMPLETED\",\n \"PREVIEW_CREATED\", \"PREVIEW_ACCEPTED\", \"PREVIEW_DISCARDED\",\n \"SCHEDULE_CREATED\", \"SCHEDULE_UPDATED\", \"SCHEDULE_DELETED\",\n \"METRICS_SNAPSHOT_CREATED\",\n}\n\nDEFAULT_RETENTION_DAYS = 90\n\n\n# #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants]\n# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning.\n# @PRE Database session is available.\n# @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot.\n# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events.\n# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent]\n# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id.\nclass TranslationEventLog:\n\n def __init__(self, db: Session):\n self.db = db\n\n # #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create]\n # @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves.\n # @RELATION DEPENDS_ON -> [TranslationEvent]\n def _create_event_raw(\n self,\n job_id: str,\n event_type: str,\n payload: Optional[Dict[str, Any]] = None,\n run_id: Optional[str] = None,\n created_by: Optional[str] = None,\n ) -> TranslationEvent:\n event = TranslationEvent(\n id=str(uuid.uuid4()),\n job_id=job_id,\n run_id=run_id,\n event_type=event_type,\n event_data=payload or {},\n created_by=created_by,\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(event)\n self.db.flush()\n return event\n # #endregion _create_event_raw\n\n # #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log]\n # @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id.\n # @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants.\n # @POST TranslationEvent row is created; invariants checked before write.\n # @SIDE_EFFECT DB write.\n def log_event(\n self,\n job_id: str,\n event_type: str,\n payload: Optional[Dict[str, Any]] = None,\n run_id: Optional[str] = None,\n created_by: Optional[str] = None,\n ) -> TranslationEvent:\n with belief_scope(\"TranslationEventLog.log_event\"):\n if event_type not in VALID_EVENT_TYPES:\n raise ValueError(\n f\"Invalid event_type '{event_type}'. \"\n f\"Must be one of: {', '.join(sorted(VALID_EVENT_TYPES))}\"\n )\n\n # Enforce terminal event invariant for non-null run_id\n if run_id is not None and event_type in TERMINAL_EVENT_TYPES:\n existing_terminal = (\n self.db.query(TranslationEvent.id)\n .filter(\n TranslationEvent.run_id == run_id,\n TranslationEvent.event_type.in_(TERMINAL_EVENT_TYPES),\n )\n .first()\n )\n if existing_terminal:\n existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id)\n raise ValueError(\n f\"Run '{run_id}' already has a terminal event \"\n f\"({existing_id}). Cannot add another terminal event.\"\n )\n\n # Enforce run_started invariant — must exist before other run events\n if run_id is not None and event_type not in {\"RUN_STARTED\"} | TERMINAL_EVENT_TYPES:\n has_started = (\n self.db.query(TranslationEvent.id)\n .filter(\n TranslationEvent.run_id == run_id,\n TranslationEvent.event_type == \"RUN_STARTED\",\n )\n .first()\n )\n if not has_started:\n raise ValueError(\n f\"Cannot log '{event_type}' for run '{run_id}': \"\n f\"RUN_STARTED event must precede other run events.\"\n )\n\n event = self._create_event_raw(\n job_id=job_id,\n event_type=event_type,\n payload=payload,\n run_id=run_id,\n created_by=created_by,\n )\n log.reason(\"Event logged\", payload={\n \"event_id\": event.id,\n \"job_id\": job_id,\n \"run_id\": run_id,\n \"event_type\": event_type,\n })\n return event\n # #endregion log_event\n\n # #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query]\n # @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination.\n def query_events(\n self,\n job_id: Optional[str] = None,\n run_id: Optional[str] = None,\n event_type: Optional[str] = None,\n limit: int = 100,\n offset: int = 0,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationEventLog.query_events\"):\n query = self.db.query(TranslationEvent)\n\n if job_id:\n query = query.filter(TranslationEvent.job_id == job_id)\n if run_id:\n query = query.filter(TranslationEvent.run_id == run_id)\n if event_type:\n query = query.filter(TranslationEvent.event_type == event_type)\n\n events = (\n query.order_by(TranslationEvent.created_at.desc())\n .offset(offset)\n .limit(limit)\n .all()\n )\n\n return [\n {\n \"id\": e.id,\n \"job_id\": e.job_id,\n \"run_id\": e.run_id,\n \"event_type\": e.event_type,\n \"event_data\": e.event_data or {},\n \"created_by\": e.created_by,\n \"created_at\": e.created_at.isoformat() if e.created_at else None,\n }\n for e in events\n ]\n # #endregion query_events\n\n # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]\n # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.\n # @PRE None.\n # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.\n # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.\n def prune_expired(\n self,\n retention_days: int = DEFAULT_RETENTION_DAYS,\n batch_size: int = 1000,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.prune_expired\"):\n cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)\n log.reason(\"Pruning expired events\", payload={\n \"cutoff\": cutoff.isoformat(),\n \"retention_days\": retention_days,\n })\n\n # Find events to prune\n expired_query = self.db.query(TranslationEvent).filter(\n TranslationEvent.created_at < cutoff\n )\n total_expired = expired_query.count()\n\n if total_expired == 0:\n log.reflect(\"No expired events to prune\", payload={})\n return {\"pruned\": 0, \"snapshot_id\": None}\n\n # Create MetricSnapshot before pruning\n snapshot = MetricSnapshot(\n id=str(uuid.uuid4()),\n job_id=\"_prune_aggregate_\",\n key_hash=f\"prune_{cutoff.timestamp():.0f}\",\n covers_events_before=cutoff,\n total_records=total_expired,\n snapshot_date=datetime.now(timezone.utc),\n )\n self.db.add(snapshot)\n self.db.flush()\n snapshot_id = snapshot.id\n\n # Delete in batches\n pruned = 0\n while True:\n batch_ids = (\n self.db.query(TranslationEvent.id)\n .filter(TranslationEvent.created_at < cutoff)\n .limit(batch_size)\n .all()\n )\n if not batch_ids:\n break\n ids = [row[0] for row in batch_ids]\n self.db.query(TranslationEvent).filter(\n TranslationEvent.id.in_(ids)\n ).delete(synchronize_session=False)\n self.db.flush()\n pruned += len(ids)\n log.reason(\"Pruned batch\", payload={\"batch_size\": len(ids), \"total_pruned\": pruned})\n\n self.db.commit()\n\n log.reflect(\"Pruning complete\", payload={\n \"pruned\": pruned,\n \"snapshot_id\": snapshot_id,\n })\n return {\"pruned\": pruned, \"snapshot_id\": snapshot_id}\n # #endregion prune_expired\n\n # #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary]\n # @BRIEF Get a summary of events for a run, including invariant validity check.\n # @RELATION DEPENDS_ON -> [TranslationEvent]\n def get_run_event_summary(self, run_id: str) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.get_run_event_summary\"):\n events = self.query_events(run_id=run_id)\n\n has_started = any(e[\"event_type\"] == \"RUN_STARTED\" for e in events)\n terminal_events = [e for e in events if e[\"event_type\"] in TERMINAL_EVENT_TYPES]\n\n return {\n \"run_id\": run_id,\n \"event_count\": len(events),\n \"has_run_started\": has_started,\n \"terminal_event_count\": len(terminal_events),\n \"terminal_events\": terminal_events,\n \"invariant_valid\": has_started and len(terminal_events) <= 1,\n \"events\": events,\n }\n # #endregion get_run_event_summary\n\n # #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants]\n # @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance.\n # @RELATION DEPENDS_ON -> [TranslationEvent]\n def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.get_run_event_invariants_lightweight\"):\n # Lightweight query: only fetch event_type column, not event_data\n event_types = [\n row[0]\n for row in (\n self.db.query(TranslationEvent.event_type)\n .filter(TranslationEvent.run_id == run_id)\n .order_by(TranslationEvent.created_at.desc())\n .limit(100)\n .all()\n )\n ]\n\n has_run_started = \"RUN_STARTED\" in event_types\n terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES)\n\n return {\n \"has_run_started\": has_run_started,\n \"terminal_event_count\": terminal_count,\n \"invariant_valid\": has_run_started and terminal_count <= 1,\n }\n # #endregion get_run_event_invariants_lightweight\n\n\n# #endregion TranslationEventLog\n# #endregion TranslationEventLog\n" }, { "contract_id": "_create_event_raw", "contract_type": "Function", "file_path": "backend/src/plugins/translate/events.py", - "start_line": 51, - "end_line": 74, + "start_line": 54, + "end_line": 77, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -64345,8 +65354,8 @@ "contract_id": "query_events", "contract_type": "Function", "file_path": "backend/src/plugins/translate/events.py", - "start_line": 145, - "end_line": 184, + "start_line": 148, + "end_line": 187, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -64378,8 +65387,8 @@ "contract_id": "prune_expired", "contract_type": "Function", "file_path": "backend/src/plugins/translate/events.py", - "start_line": 186, - "end_line": 252, + "start_line": 189, + "end_line": 255, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64417,14 +65426,14 @@ } ], "anchor_syntax": "region", - "body": " # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]\n # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.\n # @PRE None.\n # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.\n # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.\n def prune_expired(\n self,\n retention_days: int = DEFAULT_RETENTION_DAYS,\n batch_size: int = 1000,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.prune_expired\"):\n cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)\n logger.reason(\"Pruning expired events\", {\n \"cutoff\": cutoff.isoformat(),\n \"retention_days\": retention_days,\n })\n\n # Find events to prune\n expired_query = self.db.query(TranslationEvent).filter(\n TranslationEvent.created_at < cutoff\n )\n total_expired = expired_query.count()\n\n if total_expired == 0:\n logger.reflect(\"No expired events to prune\", {})\n return {\"pruned\": 0, \"snapshot_id\": None}\n\n # Create MetricSnapshot before pruning\n snapshot = MetricSnapshot(\n id=str(uuid.uuid4()),\n job_id=\"_prune_aggregate_\",\n key_hash=f\"prune_{cutoff.timestamp():.0f}\",\n covers_events_before=cutoff,\n total_records=total_expired,\n snapshot_date=datetime.now(timezone.utc),\n )\n self.db.add(snapshot)\n self.db.flush()\n snapshot_id = snapshot.id\n\n # Delete in batches\n pruned = 0\n while True:\n batch_ids = (\n self.db.query(TranslationEvent.id)\n .filter(TranslationEvent.created_at < cutoff)\n .limit(batch_size)\n .all()\n )\n if not batch_ids:\n break\n ids = [row[0] for row in batch_ids]\n self.db.query(TranslationEvent).filter(\n TranslationEvent.id.in_(ids)\n ).delete(synchronize_session=False)\n self.db.flush()\n pruned += len(ids)\n logger.reason(\"Pruned batch\", {\"batch_size\": len(ids), \"total_pruned\": pruned})\n\n self.db.commit()\n\n logger.reflect(\"Pruning complete\", {\n \"pruned\": pruned,\n \"snapshot_id\": snapshot_id,\n })\n return {\"pruned\": pruned, \"snapshot_id\": snapshot_id}\n # #endregion prune_expired\n" + "body": " # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune]\n # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail.\n # @PRE None.\n # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch.\n # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches.\n def prune_expired(\n self,\n retention_days: int = DEFAULT_RETENTION_DAYS,\n batch_size: int = 1000,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationEventLog.prune_expired\"):\n cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)\n log.reason(\"Pruning expired events\", payload={\n \"cutoff\": cutoff.isoformat(),\n \"retention_days\": retention_days,\n })\n\n # Find events to prune\n expired_query = self.db.query(TranslationEvent).filter(\n TranslationEvent.created_at < cutoff\n )\n total_expired = expired_query.count()\n\n if total_expired == 0:\n log.reflect(\"No expired events to prune\", payload={})\n return {\"pruned\": 0, \"snapshot_id\": None}\n\n # Create MetricSnapshot before pruning\n snapshot = MetricSnapshot(\n id=str(uuid.uuid4()),\n job_id=\"_prune_aggregate_\",\n key_hash=f\"prune_{cutoff.timestamp():.0f}\",\n covers_events_before=cutoff,\n total_records=total_expired,\n snapshot_date=datetime.now(timezone.utc),\n )\n self.db.add(snapshot)\n self.db.flush()\n snapshot_id = snapshot.id\n\n # Delete in batches\n pruned = 0\n while True:\n batch_ids = (\n self.db.query(TranslationEvent.id)\n .filter(TranslationEvent.created_at < cutoff)\n .limit(batch_size)\n .all()\n )\n if not batch_ids:\n break\n ids = [row[0] for row in batch_ids]\n self.db.query(TranslationEvent).filter(\n TranslationEvent.id.in_(ids)\n ).delete(synchronize_session=False)\n self.db.flush()\n pruned += len(ids)\n log.reason(\"Pruned batch\", payload={\"batch_size\": len(ids), \"total_pruned\": pruned})\n\n self.db.commit()\n\n log.reflect(\"Pruning complete\", payload={\n \"pruned\": pruned,\n \"snapshot_id\": snapshot_id,\n })\n return {\"pruned\": pruned, \"snapshot_id\": snapshot_id}\n # #endregion prune_expired\n" }, { "contract_id": "get_run_event_summary", "contract_type": "Function", "file_path": "backend/src/plugins/translate/events.py", - "start_line": 254, - "end_line": 273, + "start_line": 257, + "end_line": 276, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -64463,8 +65472,8 @@ "contract_id": "get_run_event_invariants_lightweight", "contract_type": "Function", "file_path": "backend/src/plugins/translate/events.py", - "start_line": 275, - "end_line": 300, + "start_line": 278, + "end_line": 303, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -64504,7 +65513,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/translate/executor.py", "start_line": 1, - "end_line": 774, + "end_line": 781, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -64575,14 +65584,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]\n# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationBatch]\n# @RELATION DEPENDS_ON -> [TranslationRecord]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @RELATION DEPENDS_ON -> [LLMProviderService]\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @PRE Valid TranslationRun with job configuration. DB session is available.\n# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.\n# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.\n# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]\n# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.\n# @RATIONALE Batch processing with retry — independent batches allow partial recovery.\n# @REJECTED Single monolithic LLM call — would lose all progress on any failure.\n\nimport json\nimport time\nimport uuid\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional, Set, Tuple, Callable\n\nfrom sqlalchemy.orm import Session\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.config_manager import ConfigManager\nfrom ...models.translate import (\n TranslationJob,\n TranslationRun,\n TranslationBatch,\n TranslationRecord,\n TranslationPreviewSession,\n TranslationPreviewRecord,\n)\nfrom ...services.llm_provider import LLMProviderService\nfrom ...services.llm_prompt_templates import render_prompt\nfrom ...core.superset_client import SupersetClient\nfrom .dictionary import DictionaryManager\nfrom .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE\n\n\n# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]\nMAX_RETRIES_PER_BATCH = 3\n# #endregion MAX_RETRIES_PER_BATCH\n\n\n# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]\n# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.\n# @PRE DB session and config manager available.\n# @POST Batches and records created with status tracking; run statistics updated.\n# @SIDE_EFFECT LLM API calls; DB writes.\n# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]\n# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.\nclass TranslationExecutor:\n\n def __init__(\n self,\n db: Session,\n config_manager: ConfigManager,\n current_user: Optional[str] = None,\n on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,\n ):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n self.on_batch_progress = on_batch_progress\n self._current_run_id: Optional[str] = None\n\n # #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]\n # @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.\n # @PRE run is in PENDING or RUNNING status with valid job config.\n # @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.\n # @SIDE_EFFECT LLM API calls; DB batch writes.\n def execute_run(\n self,\n run: TranslationRun,\n llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,\n full_translation: bool = False,\n ) -> TranslationRun:\n with belief_scope(\"TranslationExecutor.execute_run\"):\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found for run '{run.id}'\")\n\n logger.reason(\"Starting translation execution\", {\n \"run_id\": run.id,\n \"job_id\": job.id,\n \"batch_size\": job.batch_size,\n \"full_translation\": full_translation,\n })\n\n # Mark run as RUNNING\n run.status = \"RUNNING\"\n run.started_at = datetime.now(timezone.utc)\n self.db.flush()\n\n # Fetch source rows\n if full_translation:\n # Full translation: fetch ALL rows from Superset dataset\n source_rows = self._fetch_all_rows_from_superset(job)\n else:\n # Preview-based: fetch rows from the accepted preview session\n source_rows = self._fetch_source_rows(job.id, run.id)\n if not source_rows:\n logger.explore(\"No source rows to translate\", {\"run_id\": run.id})\n run.status = \"COMPLETED\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n return run\n\n total_rows = len(source_rows)\n run.total_records = total_rows\n\n # Split into batches\n batch_size = job.batch_size or 50\n batches = [\n source_rows[i:i + batch_size]\n for i in range(0, total_rows, batch_size)\n ]\n\n logger.reason(f\"Processing {len(batches)} batches\", {\n \"run_id\": run.id,\n \"total_rows\": total_rows,\n \"batch_size\": batch_size,\n })\n\n successful_records = 0\n failed_records = 0\n skipped_records = 0\n\n for batch_idx, batch_rows in enumerate(batches):\n batch_result = self._process_batch(\n job=job,\n run_id=run.id,\n batch_index=batch_idx,\n batch_rows=batch_rows,\n )\n successful_records += batch_result[\"successful\"]\n failed_records += batch_result[\"failed\"]\n skipped_records += batch_result[\"skipped\"]\n\n # Update run stats incrementally\n run.successful_records = successful_records\n run.failed_records = failed_records\n run.skipped_records = skipped_records\n self.db.flush()\n\n if self.on_batch_progress:\n self.on_batch_progress(\n run.id, batch_idx + 1, len(batches),\n successful_records, total_rows,\n )\n\n # Update final run status\n if failed_records == 0 and skipped_records == 0:\n run.status = \"COMPLETED\"\n elif successful_records == 0:\n run.status = \"FAILED\"\n else:\n run.status = \"COMPLETED\" # Partial success\n\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n logger.reflect(\"Translation execution complete\", {\n \"run_id\": run.id,\n \"status\": run.status,\n \"total\": total_rows,\n \"successful\": successful_records,\n \"failed\": failed_records,\n \"skipped\": skipped_records,\n })\n\n return run\n # #endregion execute_run\n\n # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]\n # @BRIEF Fetch source rows from the accepted preview session for this job.\n # @PRE job_id exists.\n # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).\n # @SIDE_EFFECT Queries preview session and records from DB.\n def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_source_rows\"):\n # Get the latest APPLIED preview session\n session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job_id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not session:\n logger.explore(\"No accepted preview session found\", {\"job_id\": job_id})\n return []\n\n # Fetch APPROVED or all records from the session\n records = (\n self.db.query(TranslationPreviewRecord)\n .filter(\n TranslationPreviewRecord.session_id == session.id,\n TranslationPreviewRecord.status.in_([\"APPROVED\", \"PENDING\"]),\n )\n .all()\n )\n\n source_rows = []\n for rec in records:\n source_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": rec.target_sql if rec.status == \"APPROVED\" else None,\n \"source_object_name\": rec.source_object_name or \"\",\n \"source_data\": rec.source_data or {},\n })\n\n logger.reason(f\"Fetched {len(source_rows)} source rows from preview\", {\n \"run_id\": run_id,\n \"session_id\": session.id,\n })\n return source_rows\n # #endregion _fetch_source_rows\n\n # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]\n # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).\n # @PRE job has source_datasource_id and environment_id configured.\n # @POST Returns list of row dicts with row_index, source_text, source_data.\n # @SIDE_EFFECT Calls Superset chart data endpoint (paginated).\n def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_all_rows_from_superset\"):\n env_id = job.environment_id or job.source_dialect or \"\"\n environments = self.config_manager.get_environments()\n env_config = next(\n (e for e in environments if e.id == env_id),\n None,\n )\n if not env_config:\n raise ValueError(f\"Superset environment '{env_id}' not found\")\n\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))\n\n # Build query context (same as preview, but with large row_limit)\n query_context = client.build_dataset_preview_query_context(\n dataset_id=int(job.source_datasource_id),\n dataset_record=dataset_detail,\n template_params={},\n effective_filters=[],\n )\n\n queries = query_context.get(\"queries\", [])\n if queries:\n queries[0][\"row_limit\"] = 100000 # Superset max default\n queries[0].pop(\"result_type\", None)\n queries[0].pop(\"columns\", None)\n queries[0][\"metrics\"] = []\n queries[0][\"row_offset\"] = 0\n query_context[\"result_type\"] = \"samples\"\n form_data = query_context.get(\"form_data\", {})\n form_data.pop(\"query_mode\", None)\n\n all_rows: List[Dict[str, Any]] = []\n batch_size_fetch = 10000 # Fetch 10K rows per pagination request\n\n while True:\n # Set pagination for this batch\n if queries:\n queries[0][\"row_limit\"] = min(batch_size_fetch, 100000)\n queries[0][\"row_offset\"] = len(all_rows)\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/chart/data\",\n data=json.dumps(query_context),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n logger.explore(\"Chart data API failed during full fetch\", {\n \"offset\": len(all_rows),\n \"error\": str(e),\n })\n if not all_rows:\n raise ValueError(f\"Failed to fetch data from Superset: {e}\")\n break # Return what we have\n\n from .preview import TranslationPreview\n rows = TranslationPreview._extract_data_rows(response)\n if not rows:\n break # No more data\n\n all_rows.extend(rows)\n logger.reason(f\"Fetched {len(rows)} rows (total: {len(all_rows)})\", {\n \"offset\": len(all_rows) - len(rows),\n })\n\n if len(rows) < batch_size_fetch:\n break # Last page\n\n # Convert to the format expected by _process_batch\n source_rows = []\n for idx, row in enumerate(all_rows):\n translation_value = str(row.get(job.translation_column, \"\") or \"\")\n context_values = {}\n if job.context_columns:\n for col in job.context_columns:\n context_values[col] = str(row.get(col, \"\") or \"\")\n # Also include target_key_cols values in context_data\n if job.target_key_cols:\n for col in job.target_key_cols:\n context_values[col] = str(row.get(col, \"\") or \"\")\n source_rows.append({\n \"row_index\": str(idx),\n \"source_text\": translation_value,\n \"source_data\": context_values,\n \"source_object_name\": f\"Row {idx + 1}\",\n \"approved_translation\": None,\n })\n\n logger.reason(f\"Prepared {len(source_rows)} source rows for full translation\", {\n \"job_id\": job.id,\n })\n return source_rows\n # #endregion _fetch_all_rows_from_superset\n\n # #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]\n # @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.\n # @PRE job and batch_rows are valid.\n # @POST TranslationBatch and TranslationRecord rows are created.\n # @SIDE_EFFECT LLM API call; DB writes.\n def _process_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_index: int,\n batch_rows: List[Dict[str, Any]],\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._process_batch\"):\n batch_start = time.monotonic()\n\n # Create batch record\n batch = TranslationBatch(\n id=str(uuid.uuid4()),\n run_id=run_id,\n batch_index=batch_index,\n status=\"RUNNING\",\n total_records=len(batch_rows),\n started_at=datetime.now(timezone.utc),\n )\n self.db.add(batch)\n self.db.flush()\n batch_id = batch.id\n\n result = {\"successful\": 0, \"failed\": 0, \"skipped\": 0, \"retries\": 0}\n\n # Extract source texts for dict filtering\n source_texts = [\n row.get(\"source_text\", \"\")\n for row in batch_rows\n if row.get(\"source_text\")\n ]\n\n # Filter dictionary entries\n dict_matches = DictionaryManager.filter_for_batch(\n self.db, source_texts, job.id\n )\n\n # For each row, determine if we need LLM translation or can use approved translation\n rows_for_llm = []\n pre_translated = []\n\n for row in batch_rows:\n if row.get(\"approved_translation\"):\n pre_translated.append(row)\n else:\n rows_for_llm.append(row)\n\n # Handle pre-translated (approved) rows\n for row in pre_translated:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=row.get(\"approved_translation\"),\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n result[\"successful\"] += 1\n\n # Process rows needing LLM translation\n if rows_for_llm:\n llm_result = self._call_llm_for_batch(\n job=job,\n run_id=run_id,\n batch_rows=rows_for_llm,\n dict_matches=dict_matches,\n batch_id=batch_id,\n )\n result[\"successful\"] += llm_result[\"successful\"]\n result[\"failed\"] += llm_result[\"failed\"]\n result[\"skipped\"] += llm_result[\"skipped\"]\n result[\"retries\"] += llm_result.get(\"retries\", 0)\n\n # Update batch status\n batch.successful_records = result[\"successful\"]\n batch.failed_records = result[\"failed\"]\n batch.completed_at = datetime.now(timezone.utc)\n batch.status = \"COMPLETED\" if result[\"failed\"] == 0 else \"COMPLETED_WITH_ERRORS\"\n self.db.flush()\n\n batch_latency = int((time.monotonic() - batch_start) * 1000)\n logger.reason(f\"Batch {batch_index} complete\", {\n \"batch_id\": batch_id,\n \"latency_ms\": batch_latency,\n **result,\n })\n\n return result\n # #endregion _process_batch\n\n # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]\n # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.\n # @PRE job has valid provider_id. batch_rows is non-empty.\n # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.\n # @SIDE_EFFECT HTTP call to LLM provider.\n def _call_llm_for_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_rows: List[Dict[str, Any]],\n dict_matches: List[Dict[str, Any]],\n batch_id: str,\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._call_llm_for_batch\"):\n # Build dictionary section\n dictionary_section = \"\"\n if dict_matches:\n glossary_lines = []\n for m in dict_matches:\n glossary_lines.append(\n f\"- '{m['source_term']}' -> '{m['target_term']}'\"\n f\"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}\"\n )\n dictionary_section = (\n \"Terminology dictionary (use these translations when applicable):\\n\"\n + \"\\n\".join(glossary_lines)\n + \"\\n\\n\"\n )\n\n # Build rows JSON for LLM\n rows_json = json.dumps([\n {\n \"row_id\": str(row.get(\"row_index\", idx)),\n \"text\": row.get(\"source_text\", \"\"),\n }\n for idx, row in enumerate(batch_rows)\n ], indent=2)\n\n # Build prompt\n prompt = render_prompt(\n DEFAULT_EXECUTION_PROMPT_TEMPLATE,\n {\n \"source_language\": job.source_dialect or \"SQL\",\n \"target_language\": job.target_language or job.target_dialect or \"en\",\n \"source_dialect\": job.source_dialect or \"\",\n \"target_dialect\": job.target_dialect or \"\",\n \"translation_column\": job.translation_column or \"\",\n \"dictionary_section\": dictionary_section,\n \"rows_json\": rows_json,\n \"row_count\": str(len(batch_rows)),\n },\n )\n\n # Call LLM with retry\n llm_response = None\n last_error = None\n retries = 0\n\n for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):\n try:\n llm_response = self._call_llm(job, prompt)\n break\n except Exception as e:\n last_error = str(e)\n retries += 1\n logger.explore(f\"LLM call failed (attempt {attempt})\", {\n \"batch_id\": batch_id,\n \"error\": last_error,\n \"attempt\": attempt,\n })\n if attempt < MAX_RETRIES_PER_BATCH:\n time.sleep(2 ** attempt) # Exponential backoff\n else:\n logger.explore(\"LLM call exhausted retries\", {\n \"batch_id\": batch_id,\n \"last_error\": last_error,\n })\n\n if llm_response is None:\n # All retries failed — mark all rows as failed\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"FAILED\",\n error_message=f\"LLM call failed after {retries} retries: {last_error}\",\n )\n self.db.add(record)\n return {\"successful\": 0, \"failed\": len(batch_rows), \"skipped\": 0, \"retries\": retries}\n\n # Parse LLM response\n try:\n translations = self._parse_llm_response(llm_response, len(batch_rows))\n except ValueError as e:\n # Parse failure — mark all rows as SKIPPED\n logger.explore(\"LLM response parse failed\", {\n \"batch_id\": batch_id,\n \"error\": str(e),\n \"response_preview\": llm_response[:500] if llm_response else \"\",\n })\n skipped = len(batch_rows)\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=f\"LLM parse failure: {e}\",\n )\n self.db.add(record)\n return {\n \"successful\": 0,\n \"failed\": 0,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n\n successful = 0\n failed = 0\n skipped = 0\n\n for row in batch_rows:\n row_id = str(row.get(\"row_index\", \"\"))\n translation = translations.get(row_id)\n\n if translation is None:\n # NULL translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"NULL translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n if translation.strip() == \"\":\n # Empty translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"Empty translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n successful += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=translation,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n\n return {\n \"successful\": successful,\n \"failed\": failed,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n # #endregion _call_llm_for_batch\n\n # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]\n # @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.\n # @PRE job has valid provider_id.\n # @POST Returns raw LLM response string.\n # @SIDE_EFFECT HTTP call to LLM provider.\n def _call_llm(self, job: TranslationJob, prompt: str) -> str:\n with belief_scope(\"TranslationExecutor._call_llm\"):\n if not job.provider_id:\n raise ValueError(\"Job has no LLM provider configured\")\n\n provider_svc = LLMProviderService(self.db)\n provider = provider_svc.get_provider(job.provider_id)\n if not provider:\n raise ValueError(f\"LLM provider '{job.provider_id}' not found\")\n\n api_key = provider_svc.get_decrypted_api_key(job.provider_id)\n if not api_key:\n raise ValueError(f\"Could not decrypt API key for provider '{job.provider_id}'\")\n\n model = provider.default_model or \"gpt-4o-mini\"\n provider_type = provider.provider_type.lower() if provider.provider_type else \"openai\"\n\n if provider_type in (\"openai\", \"openai_compatible\", \"openrouter\", \"kilo\"):\n return self._call_openai_compatible(\n base_url=provider.base_url,\n api_key=api_key,\n model=model,\n prompt=prompt,\n provider_type=provider_type,\n )\n else:\n raise ValueError(f\"Unsupported provider type '{provider_type}'\")\n # #endregion _call_llm\n\n # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]\n # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.\n # @PRE Valid API endpoint, key, model, and prompt.\n # @POST Returns response text.\n # @SIDE_EFFECT HTTP POST to LLM API.\n @staticmethod\n def _call_openai_compatible(\n base_url: str,\n api_key: str,\n model: str,\n prompt: str,\n provider_type: str = \"openai\",\n ) -> str:\n with belief_scope(\"TranslationExecutor._call_openai_compatible\"):\n import requests as http_requests\n\n url = f\"{base_url.rstrip('/')}/chat/completions\"\n headers = {\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n }\n payload = {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a database content translation assistant. Translate the provided text accurately, preserving data semantics.\"},\n {\"role\": \"user\", \"content\": prompt},\n ],\n \"temperature\": 0.1,\n \"max_tokens\": 4096,\n }\n # Structured output (response_format) only for native OpenAI — upstream providers routed via\n # Kilo/OpenRouter may not support it (e.g. StepFun returns \"structured_outputs is not supported\")\n if provider_type in (\"openai\", \"openai_compatible\"):\n payload[\"response_format\"] = {\"type\": \"json_object\"}\n\n logger.reason(\n f\"LLM request model={payload.get('model')} \"\n f\"provider_type={provider_type} \"\n f\"response_format={'yes' if 'response_format' in payload else 'no'} \"\n f\"prompt_len={len(prompt)}\"\n )\n response = http_requests.post(url, headers=headers, json=payload, timeout=180)\n if not response.ok:\n logger.explore(\n f\"LLM API error status={response.status_code} \"\n f\"model={payload.get('model')} \"\n f\"body={response.text[:2000]}\"\n )\n response.raise_for_status()\n data = response.json()\n\n choices = data.get(\"choices\", [])\n if not choices:\n raise ValueError(\"LLM returned no choices\")\n\n content = choices[0].get(\"message\", {}).get(\"content\", \"\")\n if not content:\n # Log full response for diagnostics\n finish_reason = choices[0].get(\"finish_reason\", \"unknown\")\n logger.explore(\"LLM returned empty content\", {\n \"finish_reason\": finish_reason,\n \"model\": payload.get(\"model\"),\n \"prompt_len\": len(prompt),\n \"response_keys\": list(data.keys()),\n \"choices_count\": len(choices),\n })\n raise ValueError(\n f\"LLM returned empty content (finish_reason={finish_reason}, \"\n f\"model={payload.get('model')}, prompt_len={len(prompt)})\"\n )\n\n return content\n # #endregion _call_openai_compatible\n\n # #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]\n # @BRIEF Parse LLM JSON response into dict of row_id -> translation text.\n # @RELATION DEPENDS_ON -> [json]\n @staticmethod\n def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:\n with belief_scope(\"TranslationExecutor._parse_llm_response\"):\n try:\n data = json.loads(response_text)\n except json.JSONDecodeError:\n # Try to extract from markdown code block\n import re\n match = re.search(r'```(?:json)?\\s*\\n?(.*?)\\n?```', response_text, re.DOTALL)\n if match:\n try:\n data = json.loads(match.group(1))\n except json.JSONDecodeError:\n raise ValueError(\"LLM response was not valid JSON\")\n else:\n raise ValueError(\"LLM response was not valid JSON\")\n\n rows = data.get(\"rows\", [])\n if not isinstance(rows, list):\n raise ValueError(\"LLM response missing 'rows' array\")\n\n translations: Dict[str, str] = {}\n for item in rows:\n row_id = str(item.get(\"row_id\", \"\"))\n translation = item.get(\"translation\")\n if translation is None:\n # Skip NULL translations — they'll be handled by caller\n continue\n if row_id:\n translations[row_id] = str(translation)\n\n return translations\n # #endregion _parse_llm_response\n\n\n# #endregion TranslationExecutor\n# #endregion TranslationExecutor\n" + "body": "# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm]\n# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationBatch]\n# @RELATION DEPENDS_ON -> [TranslationRecord]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @RELATION DEPENDS_ON -> [LLMProviderService]\n# @RELATION DEPENDS_ON -> [DictionaryManager]\n# @RELATION DEPENDS_ON -> [TranslationPreview]\n# @PRE Valid TranslationRun with job configuration. DB session is available.\n# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated.\n# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics.\n# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records]\n# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3.\n# @RATIONALE Batch processing with retry — independent batches allow partial recovery.\n# @REJECTED Single monolithic LLM call — would lose all progress on any failure.\n\nimport json\nimport time\nimport uuid\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional, Set, Tuple, Callable\n\nfrom sqlalchemy.orm import Session\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.config_manager import ConfigManager\nfrom ...models.translate import (\n TranslationJob,\n TranslationRun,\n TranslationBatch,\n TranslationRecord,\n TranslationPreviewSession,\n TranslationPreviewRecord,\n)\nfrom ...services.llm_provider import LLMProviderService\nfrom ...services.llm_prompt_templates import render_prompt\nfrom ...core.superset_client import SupersetClient\nfrom .dictionary import DictionaryManager\nfrom .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE\n\nlog = MarkerLogger(\"TranslationExecutor\")\n\n# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry]\nMAX_RETRIES_PER_BATCH = 3\n# #endregion MAX_RETRIES_PER_BATCH\n\n\n# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch]\n# @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results.\n# @PRE DB session and config manager available.\n# @POST Batches and records created with status tracking; run statistics updated.\n# @SIDE_EFFECT LLM API calls; DB writes.\n# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats]\n# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS.\nclass TranslationExecutor:\n\n def __init__(\n self,\n db: Session,\n config_manager: ConfigManager,\n current_user: Optional[str] = None,\n on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,\n ):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n self.on_batch_progress = on_batch_progress\n self._current_run_id: Optional[str] = None\n\n # #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute]\n # @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist.\n # @PRE run is in PENDING or RUNNING status with valid job config.\n # @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED.\n # @SIDE_EFFECT LLM API calls; DB batch writes.\n def execute_run(\n self,\n run: TranslationRun,\n llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,\n full_translation: bool = False,\n ) -> TranslationRun:\n with belief_scope(\"TranslationExecutor.execute_run\"):\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found for run '{run.id}'\")\n\n log.reason(\"Starting translation execution\", payload={\n \"run_id\": run.id,\n \"job_id\": job.id,\n \"batch_size\": job.batch_size,\n \"full_translation\": full_translation,\n })\n\n # Mark run as RUNNING\n run.status = \"RUNNING\"\n run.started_at = datetime.now(timezone.utc)\n self.db.flush()\n\n # Fetch source rows\n if full_translation:\n # Full translation: fetch ALL rows from Superset dataset\n source_rows = self._fetch_all_rows_from_superset(job)\n else:\n # Preview-based: fetch rows from the accepted preview session\n source_rows = self._fetch_source_rows(job.id, run.id)\n if not source_rows:\n log.explore(\"No source rows to translate\", payload={\"run_id\": run.id}, error=\"Preview produced 0 source rows for translation\")\n run.status = \"COMPLETED\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n return run\n\n total_rows = len(source_rows)\n run.total_records = total_rows\n\n # Split into batches\n batch_size = job.batch_size or 50\n batches = [\n source_rows[i:i + batch_size]\n for i in range(0, total_rows, batch_size)\n ]\n\n log.reason(f\"Processing {len(batches)} batches\", payload={\n \"run_id\": run.id,\n \"total_rows\": total_rows,\n \"batch_size\": batch_size,\n })\n\n successful_records = 0\n failed_records = 0\n skipped_records = 0\n\n for batch_idx, batch_rows in enumerate(batches):\n batch_result = self._process_batch(\n job=job,\n run_id=run.id,\n batch_index=batch_idx,\n batch_rows=batch_rows,\n )\n successful_records += batch_result[\"successful\"]\n failed_records += batch_result[\"failed\"]\n skipped_records += batch_result[\"skipped\"]\n\n # Update run stats incrementally\n run.successful_records = successful_records\n run.failed_records = failed_records\n run.skipped_records = skipped_records\n self.db.flush()\n\n if self.on_batch_progress:\n self.on_batch_progress(\n run.id, batch_idx + 1, len(batches),\n successful_records, total_rows,\n )\n\n # Update final run status\n if failed_records == 0 and skipped_records == 0:\n run.status = \"COMPLETED\"\n elif successful_records == 0:\n run.status = \"FAILED\"\n else:\n run.status = \"COMPLETED\" # Partial success\n\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n log.reflect(\"Translation execution complete\", payload={\n \"run_id\": run.id,\n \"status\": run.status,\n \"total\": total_rows,\n \"successful\": successful_records,\n \"failed\": failed_records,\n \"skipped\": skipped_records,\n })\n\n return run\n # #endregion execute_run\n\n # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]\n # @BRIEF Fetch source rows from the accepted preview session for this job.\n # @PRE job_id exists.\n # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).\n # @SIDE_EFFECT Queries preview session and records from DB.\n def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_source_rows\"):\n # Get the latest APPLIED preview session\n session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job_id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not session:\n log.explore(\"No accepted preview session found\", error=\"Preview session has no accepted rows\", payload={\"job_id\": job_id})\n return []\n\n # Fetch APPROVED or all records from the session\n records = (\n self.db.query(TranslationPreviewRecord)\n .filter(\n TranslationPreviewRecord.session_id == session.id,\n TranslationPreviewRecord.status.in_([\"APPROVED\", \"PENDING\"]),\n )\n .all()\n )\n\n source_rows = []\n for rec in records:\n source_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": rec.target_sql if rec.status == \"APPROVED\" else None,\n \"source_object_name\": rec.source_object_name or \"\",\n \"source_data\": rec.source_data or {},\n })\n\n log.reason(f\"Fetched {len(source_rows)} source rows from preview\", payload={\n \"run_id\": run_id,\n \"session_id\": session.id,\n })\n return source_rows\n # #endregion _fetch_source_rows\n\n # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]\n # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).\n # @PRE job has source_datasource_id and environment_id configured.\n # @POST Returns list of row dicts with row_index, source_text, source_data.\n # @SIDE_EFFECT Calls Superset chart data endpoint (paginated).\n def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_all_rows_from_superset\"):\n env_id = job.environment_id or job.source_dialect or \"\"\n environments = self.config_manager.get_environments()\n env_config = next(\n (e for e in environments if e.id == env_id),\n None,\n )\n if not env_config:\n raise ValueError(f\"Superset environment '{env_id}' not found\")\n\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))\n\n # Build query context (same as preview, but with large row_limit)\n query_context = client.build_dataset_preview_query_context(\n dataset_id=int(job.source_datasource_id),\n dataset_record=dataset_detail,\n template_params={},\n effective_filters=[],\n )\n\n queries = query_context.get(\"queries\", [])\n if queries:\n queries[0][\"row_limit\"] = 100000 # Superset max default\n queries[0].pop(\"result_type\", None)\n queries[0].pop(\"columns\", None)\n queries[0][\"metrics\"] = []\n queries[0][\"row_offset\"] = 0\n query_context[\"result_type\"] = \"samples\"\n form_data = query_context.get(\"form_data\", {})\n form_data.pop(\"query_mode\", None)\n\n all_rows: List[Dict[str, Any]] = []\n batch_size_fetch = 10000 # Fetch 10K rows per pagination request\n\n while True:\n # Set pagination for this batch\n if queries:\n queries[0][\"row_limit\"] = min(batch_size_fetch, 100000)\n queries[0][\"row_offset\"] = len(all_rows)\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/chart/data\",\n data=json.dumps(query_context),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n log.explore(\"Chart data API failed during full fetch\", error=\"Chart data API error\",\n payload={\n \"offset\": len(all_rows),\n \"error\": str(e),\n })\n if not all_rows:\n raise ValueError(f\"Failed to fetch data from Superset: {e}\")\n break # Return what we have\n\n from .preview import TranslationPreview\n rows = TranslationPreview._extract_data_rows(response)\n if not rows:\n break # No more data\n\n all_rows.extend(rows)\n log.reason(f\"Fetched {len(rows)} rows (total: {len(all_rows)})\", payload={\n \"offset\": len(all_rows) - len(rows),\n })\n\n if len(rows) < batch_size_fetch:\n break # Last page\n\n # Convert to the format expected by _process_batch\n source_rows = []\n for idx, row in enumerate(all_rows):\n translation_value = str(row.get(job.translation_column, \"\") or \"\")\n context_values = {}\n if job.context_columns:\n for col in job.context_columns:\n context_values[col] = str(row.get(col, \"\") or \"\")\n # Also include target_key_cols values in context_data\n if job.target_key_cols:\n for col in job.target_key_cols:\n context_values[col] = str(row.get(col, \"\") or \"\")\n source_rows.append({\n \"row_index\": str(idx),\n \"source_text\": translation_value,\n \"source_data\": context_values,\n \"source_object_name\": f\"Row {idx + 1}\",\n \"approved_translation\": None,\n })\n\n log.reason(f\"Prepared {len(source_rows)} source rows for full translation\", payload={\n \"job_id\": job.id,\n })\n return source_rows\n # #endregion _fetch_all_rows_from_superset\n\n # #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]\n # @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.\n # @PRE job and batch_rows are valid.\n # @POST TranslationBatch and TranslationRecord rows are created.\n # @SIDE_EFFECT LLM API call; DB writes.\n def _process_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_index: int,\n batch_rows: List[Dict[str, Any]],\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._process_batch\"):\n batch_start = time.monotonic()\n\n # Create batch record\n batch = TranslationBatch(\n id=str(uuid.uuid4()),\n run_id=run_id,\n batch_index=batch_index,\n status=\"RUNNING\",\n total_records=len(batch_rows),\n started_at=datetime.now(timezone.utc),\n )\n self.db.add(batch)\n self.db.flush()\n batch_id = batch.id\n\n result = {\"successful\": 0, \"failed\": 0, \"skipped\": 0, \"retries\": 0}\n\n # Extract source texts for dict filtering\n source_texts = [\n row.get(\"source_text\", \"\")\n for row in batch_rows\n if row.get(\"source_text\")\n ]\n\n # Filter dictionary entries\n dict_matches = DictionaryManager.filter_for_batch(\n self.db, source_texts, job.id\n )\n\n # For each row, determine if we need LLM translation or can use approved translation\n rows_for_llm = []\n pre_translated = []\n\n for row in batch_rows:\n if row.get(\"approved_translation\"):\n pre_translated.append(row)\n else:\n rows_for_llm.append(row)\n\n # Handle pre-translated (approved) rows\n for row in pre_translated:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=row.get(\"approved_translation\"),\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n result[\"successful\"] += 1\n\n # Process rows needing LLM translation\n if rows_for_llm:\n llm_result = self._call_llm_for_batch(\n job=job,\n run_id=run_id,\n batch_rows=rows_for_llm,\n dict_matches=dict_matches,\n batch_id=batch_id,\n )\n result[\"successful\"] += llm_result[\"successful\"]\n result[\"failed\"] += llm_result[\"failed\"]\n result[\"skipped\"] += llm_result[\"skipped\"]\n result[\"retries\"] += llm_result.get(\"retries\", 0)\n\n # Update batch status\n batch.successful_records = result[\"successful\"]\n batch.failed_records = result[\"failed\"]\n batch.completed_at = datetime.now(timezone.utc)\n batch.status = \"COMPLETED\" if result[\"failed\"] == 0 else \"COMPLETED_WITH_ERRORS\"\n self.db.flush()\n\n batch_latency = int((time.monotonic() - batch_start) * 1000)\n log.reason(f\"Batch {batch_index} complete\", payload={\n \"batch_id\": batch_id,\n \"latency_ms\": batch_latency,\n **result,\n })\n\n return result\n # #endregion _process_batch\n\n # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]\n # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.\n # @PRE job has valid provider_id. batch_rows is non-empty.\n # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.\n # @SIDE_EFFECT HTTP call to LLM provider.\n def _call_llm_for_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_rows: List[Dict[str, Any]],\n dict_matches: List[Dict[str, Any]],\n batch_id: str,\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._call_llm_for_batch\"):\n # Build dictionary section\n dictionary_section = \"\"\n if dict_matches:\n glossary_lines = []\n for m in dict_matches:\n glossary_lines.append(\n f\"- '{m['source_term']}' -> '{m['target_term']}'\"\n f\"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}\"\n )\n dictionary_section = (\n \"Terminology dictionary (use these translations when applicable):\\n\"\n + \"\\n\".join(glossary_lines)\n + \"\\n\\n\"\n )\n\n # Build rows JSON for LLM\n rows_json = json.dumps([\n {\n \"row_id\": str(row.get(\"row_index\", idx)),\n \"text\": row.get(\"source_text\", \"\"),\n }\n for idx, row in enumerate(batch_rows)\n ], indent=2)\n\n # Build prompt\n prompt = render_prompt(\n DEFAULT_EXECUTION_PROMPT_TEMPLATE,\n {\n \"source_language\": job.source_dialect or \"SQL\",\n \"target_language\": job.target_language or job.target_dialect or \"en\",\n \"source_dialect\": job.source_dialect or \"\",\n \"target_dialect\": job.target_dialect or \"\",\n \"translation_column\": job.translation_column or \"\",\n \"dictionary_section\": dictionary_section,\n \"rows_json\": rows_json,\n \"row_count\": str(len(batch_rows)),\n },\n )\n\n # Call LLM with retry\n llm_response = None\n last_error = None\n retries = 0\n\n for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):\n try:\n llm_response = self._call_llm(job, prompt)\n break\n except Exception as e:\n last_error = str(e)\n retries += 1\n log.explore(\"LLM call failed\", error=\"LLM call failed, retrying\",\n payload={\n \"batch_id\": batch_id,\n \"error\": last_error,\n \"attempt\": attempt,\n })\n if attempt < MAX_RETRIES_PER_BATCH:\n time.sleep(2 ** attempt) # Exponential backoff\n else:\n log.explore(\"LLM call exhausted retries\", error=\"LLM retries exhausted\",\n payload={\n \"batch_id\": batch_id,\n \"last_error\": last_error,\n })\n\n if llm_response is None:\n # All retries failed — mark all rows as failed\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"FAILED\",\n error_message=f\"LLM call failed after {retries} retries: {last_error}\",\n )\n self.db.add(record)\n return {\"successful\": 0, \"failed\": len(batch_rows), \"skipped\": 0, \"retries\": retries}\n\n # Parse LLM response\n try:\n translations = self._parse_llm_response(llm_response, len(batch_rows))\n except ValueError as e:\n # Parse failure — mark all rows as SKIPPED\n log.explore(\"LLM response parse failed\", error=\"Failed to parse LLM JSON response\",\n payload={\n \"batch_id\": batch_id,\n \"error\": str(e),\n \"response_preview\": llm_response[:500] if llm_response else \"\",\n })\n skipped = len(batch_rows)\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=f\"LLM parse failure: {e}\",\n )\n self.db.add(record)\n return {\n \"successful\": 0,\n \"failed\": 0,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n\n successful = 0\n failed = 0\n skipped = 0\n\n for row in batch_rows:\n row_id = str(row.get(\"row_index\", \"\"))\n translation = translations.get(row_id)\n\n if translation is None:\n # NULL translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"NULL translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n if translation.strip() == \"\":\n # Empty translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"Empty translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n successful += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=translation,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n\n return {\n \"successful\": successful,\n \"failed\": failed,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n # #endregion _call_llm_for_batch\n\n # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call]\n # @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type.\n # @PRE job has valid provider_id.\n # @POST Returns raw LLM response string.\n # @SIDE_EFFECT HTTP call to LLM provider.\n def _call_llm(self, job: TranslationJob, prompt: str) -> str:\n with belief_scope(\"TranslationExecutor._call_llm\"):\n if not job.provider_id:\n raise ValueError(\"Job has no LLM provider configured\")\n\n provider_svc = LLMProviderService(self.db)\n provider = provider_svc.get_provider(job.provider_id)\n if not provider:\n raise ValueError(f\"LLM provider '{job.provider_id}' not found\")\n\n api_key = provider_svc.get_decrypted_api_key(job.provider_id)\n if not api_key:\n raise ValueError(f\"Could not decrypt API key for provider '{job.provider_id}'\")\n\n model = provider.default_model or \"gpt-4o-mini\"\n provider_type = provider.provider_type.lower() if provider.provider_type else \"openai\"\n\n if provider_type in (\"openai\", \"openai_compatible\", \"openrouter\", \"kilo\"):\n return self._call_openai_compatible(\n base_url=provider.base_url,\n api_key=api_key,\n model=model,\n prompt=prompt,\n provider_type=provider_type,\n )\n else:\n raise ValueError(f\"Unsupported provider type '{provider_type}'\")\n # #endregion _call_llm\n\n # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]\n # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.\n # @PRE Valid API endpoint, key, model, and prompt.\n # @POST Returns response text.\n # @SIDE_EFFECT HTTP POST to LLM API.\n @staticmethod\n def _call_openai_compatible(\n base_url: str,\n api_key: str,\n model: str,\n prompt: str,\n provider_type: str = \"openai\",\n ) -> str:\n with belief_scope(\"TranslationExecutor._call_openai_compatible\"):\n import requests as http_requests\n\n url = f\"{base_url.rstrip('/')}/chat/completions\"\n headers = {\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n }\n payload = {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a database content translation assistant. Translate the provided text accurately, preserving data semantics.\"},\n {\"role\": \"user\", \"content\": prompt},\n ],\n \"temperature\": 0.1,\n \"max_tokens\": 4096,\n }\n # Structured output (response_format) only for native OpenAI — upstream providers routed via\n # Kilo/OpenRouter may not support it (e.g. StepFun returns \"structured_outputs is not supported\")\n if provider_type in (\"openai\", \"openai_compatible\"):\n payload[\"response_format\"] = {\"type\": \"json_object\"}\n\n log.reason(\n f\"LLM request model={payload.get('model')} \"\n f\"provider_type={provider_type} \"\n f\"response_format={'yes' if 'response_format' in payload else 'no'} \"\n f\"prompt_len={len(prompt)}\"\n )\n response = http_requests.post(url, headers=headers, json=payload, timeout=180)\n if not response.ok:\n log.explore(\"LLM API error\", error=f\"LLM API returned status {response.status_code}\", payload={\n \"status_code\": response.status_code,\n \"model\": payload.get('model'),\n \"body\": response.text[:2000],\n })\n response.raise_for_status()\n data = response.json()\n\n choices = data.get(\"choices\", [])\n if not choices:\n raise ValueError(\"LLM returned no choices\")\n\n content = choices[0].get(\"message\", {}).get(\"content\", \"\")\n if not content:\n # Log full response for diagnostics\n finish_reason = choices[0].get(\"finish_reason\", \"unknown\")\n log.explore(\"LLM returned empty content\", error=\"Empty response from LLM\",\n payload={\n \"finish_reason\": finish_reason,\n \"model\": payload.get(\"model\"),\n \"prompt_len\": len(prompt),\n \"response_keys\": list(data.keys()),\n \"choices_count\": len(choices),\n })\n raise ValueError(\n f\"LLM returned empty content (finish_reason={finish_reason}, \"\n f\"model={payload.get('model')}, prompt_len={len(prompt)})\"\n )\n\n return content\n # #endregion _call_openai_compatible\n\n # #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse]\n # @BRIEF Parse LLM JSON response into dict of row_id -> translation text.\n # @RELATION DEPENDS_ON -> [json]\n @staticmethod\n def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:\n with belief_scope(\"TranslationExecutor._parse_llm_response\"):\n try:\n data = json.loads(response_text)\n except json.JSONDecodeError:\n # Try to extract from markdown code block\n import re\n match = re.search(r'```(?:json)?\\s*\\n?(.*?)\\n?```', response_text, re.DOTALL)\n if match:\n try:\n data = json.loads(match.group(1))\n except json.JSONDecodeError:\n raise ValueError(\"LLM response was not valid JSON\")\n else:\n raise ValueError(\"LLM response was not valid JSON\")\n\n rows = data.get(\"rows\", [])\n if not isinstance(rows, list):\n raise ValueError(\"LLM response missing 'rows' array\")\n\n translations: Dict[str, str] = {}\n for item in rows:\n row_id = str(item.get(\"row_id\", \"\"))\n translation = item.get(\"translation\")\n if translation is None:\n # Skip NULL translations — they'll be handled by caller\n continue\n if row_id:\n translations[row_id] = str(translation)\n\n return translations\n # #endregion _parse_llm_response\n\n\n# #endregion TranslationExecutor\n# #endregion TranslationExecutor\n" }, { "contract_id": "MAX_RETRIES_PER_BATCH", "contract_type": "Constant", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 43, - "end_line": 45, + "start_line": 45, + "end_line": 47, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -64622,8 +65631,8 @@ "contract_id": "_fetch_source_rows", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 178, - "end_line": 224, + "start_line": 180, + "end_line": 226, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64661,14 +65670,14 @@ } ], "anchor_syntax": "region", - "body": " # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]\n # @BRIEF Fetch source rows from the accepted preview session for this job.\n # @PRE job_id exists.\n # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).\n # @SIDE_EFFECT Queries preview session and records from DB.\n def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_source_rows\"):\n # Get the latest APPLIED preview session\n session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job_id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not session:\n logger.explore(\"No accepted preview session found\", {\"job_id\": job_id})\n return []\n\n # Fetch APPROVED or all records from the session\n records = (\n self.db.query(TranslationPreviewRecord)\n .filter(\n TranslationPreviewRecord.session_id == session.id,\n TranslationPreviewRecord.status.in_([\"APPROVED\", \"PENDING\"]),\n )\n .all()\n )\n\n source_rows = []\n for rec in records:\n source_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": rec.target_sql if rec.status == \"APPROVED\" else None,\n \"source_object_name\": rec.source_object_name or \"\",\n \"source_data\": rec.source_data or {},\n })\n\n logger.reason(f\"Fetched {len(source_rows)} source rows from preview\", {\n \"run_id\": run_id,\n \"session_id\": session.id,\n })\n return source_rows\n # #endregion _fetch_source_rows\n" + "body": " # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows]\n # @BRIEF Fetch source rows from the accepted preview session for this job.\n # @PRE job_id exists.\n # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc).\n # @SIDE_EFFECT Queries preview session and records from DB.\n def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_source_rows\"):\n # Get the latest APPLIED preview session\n session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job_id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not session:\n log.explore(\"No accepted preview session found\", error=\"Preview session has no accepted rows\", payload={\"job_id\": job_id})\n return []\n\n # Fetch APPROVED or all records from the session\n records = (\n self.db.query(TranslationPreviewRecord)\n .filter(\n TranslationPreviewRecord.session_id == session.id,\n TranslationPreviewRecord.status.in_([\"APPROVED\", \"PENDING\"]),\n )\n .all()\n )\n\n source_rows = []\n for rec in records:\n source_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": rec.target_sql if rec.status == \"APPROVED\" else None,\n \"source_object_name\": rec.source_object_name or \"\",\n \"source_data\": rec.source_data or {},\n })\n\n log.reason(f\"Fetched {len(source_rows)} source rows from preview\", payload={\n \"run_id\": run_id,\n \"session_id\": session.id,\n })\n return source_rows\n # #endregion _fetch_source_rows\n" }, { "contract_id": "_fetch_all_rows_from_superset", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 226, - "end_line": 326, + "start_line": 228, + "end_line": 329, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64706,14 +65715,14 @@ } ], "anchor_syntax": "region", - "body": " # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]\n # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).\n # @PRE job has source_datasource_id and environment_id configured.\n # @POST Returns list of row dicts with row_index, source_text, source_data.\n # @SIDE_EFFECT Calls Superset chart data endpoint (paginated).\n def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_all_rows_from_superset\"):\n env_id = job.environment_id or job.source_dialect or \"\"\n environments = self.config_manager.get_environments()\n env_config = next(\n (e for e in environments if e.id == env_id),\n None,\n )\n if not env_config:\n raise ValueError(f\"Superset environment '{env_id}' not found\")\n\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))\n\n # Build query context (same as preview, but with large row_limit)\n query_context = client.build_dataset_preview_query_context(\n dataset_id=int(job.source_datasource_id),\n dataset_record=dataset_detail,\n template_params={},\n effective_filters=[],\n )\n\n queries = query_context.get(\"queries\", [])\n if queries:\n queries[0][\"row_limit\"] = 100000 # Superset max default\n queries[0].pop(\"result_type\", None)\n queries[0].pop(\"columns\", None)\n queries[0][\"metrics\"] = []\n queries[0][\"row_offset\"] = 0\n query_context[\"result_type\"] = \"samples\"\n form_data = query_context.get(\"form_data\", {})\n form_data.pop(\"query_mode\", None)\n\n all_rows: List[Dict[str, Any]] = []\n batch_size_fetch = 10000 # Fetch 10K rows per pagination request\n\n while True:\n # Set pagination for this batch\n if queries:\n queries[0][\"row_limit\"] = min(batch_size_fetch, 100000)\n queries[0][\"row_offset\"] = len(all_rows)\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/chart/data\",\n data=json.dumps(query_context),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n logger.explore(\"Chart data API failed during full fetch\", {\n \"offset\": len(all_rows),\n \"error\": str(e),\n })\n if not all_rows:\n raise ValueError(f\"Failed to fetch data from Superset: {e}\")\n break # Return what we have\n\n from .preview import TranslationPreview\n rows = TranslationPreview._extract_data_rows(response)\n if not rows:\n break # No more data\n\n all_rows.extend(rows)\n logger.reason(f\"Fetched {len(rows)} rows (total: {len(all_rows)})\", {\n \"offset\": len(all_rows) - len(rows),\n })\n\n if len(rows) < batch_size_fetch:\n break # Last page\n\n # Convert to the format expected by _process_batch\n source_rows = []\n for idx, row in enumerate(all_rows):\n translation_value = str(row.get(job.translation_column, \"\") or \"\")\n context_values = {}\n if job.context_columns:\n for col in job.context_columns:\n context_values[col] = str(row.get(col, \"\") or \"\")\n # Also include target_key_cols values in context_data\n if job.target_key_cols:\n for col in job.target_key_cols:\n context_values[col] = str(row.get(col, \"\") or \"\")\n source_rows.append({\n \"row_index\": str(idx),\n \"source_text\": translation_value,\n \"source_data\": context_values,\n \"source_object_name\": f\"Row {idx + 1}\",\n \"approved_translation\": None,\n })\n\n logger.reason(f\"Prepared {len(source_rows)} source rows for full translation\", {\n \"job_id\": job.id,\n })\n return source_rows\n # #endregion _fetch_all_rows_from_superset\n" + "body": " # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows]\n # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated).\n # @PRE job has source_datasource_id and environment_id configured.\n # @POST Returns list of row dicts with row_index, source_text, source_data.\n # @SIDE_EFFECT Calls Superset chart data endpoint (paginated).\n def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationExecutor._fetch_all_rows_from_superset\"):\n env_id = job.environment_id or job.source_dialect or \"\"\n environments = self.config_manager.get_environments()\n env_config = next(\n (e for e in environments if e.id == env_id),\n None,\n )\n if not env_config:\n raise ValueError(f\"Superset environment '{env_id}' not found\")\n\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))\n\n # Build query context (same as preview, but with large row_limit)\n query_context = client.build_dataset_preview_query_context(\n dataset_id=int(job.source_datasource_id),\n dataset_record=dataset_detail,\n template_params={},\n effective_filters=[],\n )\n\n queries = query_context.get(\"queries\", [])\n if queries:\n queries[0][\"row_limit\"] = 100000 # Superset max default\n queries[0].pop(\"result_type\", None)\n queries[0].pop(\"columns\", None)\n queries[0][\"metrics\"] = []\n queries[0][\"row_offset\"] = 0\n query_context[\"result_type\"] = \"samples\"\n form_data = query_context.get(\"form_data\", {})\n form_data.pop(\"query_mode\", None)\n\n all_rows: List[Dict[str, Any]] = []\n batch_size_fetch = 10000 # Fetch 10K rows per pagination request\n\n while True:\n # Set pagination for this batch\n if queries:\n queries[0][\"row_limit\"] = min(batch_size_fetch, 100000)\n queries[0][\"row_offset\"] = len(all_rows)\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/chart/data\",\n data=json.dumps(query_context),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n log.explore(\"Chart data API failed during full fetch\", error=\"Chart data API error\",\n payload={\n \"offset\": len(all_rows),\n \"error\": str(e),\n })\n if not all_rows:\n raise ValueError(f\"Failed to fetch data from Superset: {e}\")\n break # Return what we have\n\n from .preview import TranslationPreview\n rows = TranslationPreview._extract_data_rows(response)\n if not rows:\n break # No more data\n\n all_rows.extend(rows)\n log.reason(f\"Fetched {len(rows)} rows (total: {len(all_rows)})\", payload={\n \"offset\": len(all_rows) - len(rows),\n })\n\n if len(rows) < batch_size_fetch:\n break # Last page\n\n # Convert to the format expected by _process_batch\n source_rows = []\n for idx, row in enumerate(all_rows):\n translation_value = str(row.get(job.translation_column, \"\") or \"\")\n context_values = {}\n if job.context_columns:\n for col in job.context_columns:\n context_values[col] = str(row.get(col, \"\") or \"\")\n # Also include target_key_cols values in context_data\n if job.target_key_cols:\n for col in job.target_key_cols:\n context_values[col] = str(row.get(col, \"\") or \"\")\n source_rows.append({\n \"row_index\": str(idx),\n \"source_text\": translation_value,\n \"source_data\": context_values,\n \"source_object_name\": f\"Row {idx + 1}\",\n \"approved_translation\": None,\n })\n\n log.reason(f\"Prepared {len(source_rows)} source rows for full translation\", payload={\n \"job_id\": job.id,\n })\n return source_rows\n # #endregion _fetch_all_rows_from_superset\n" }, { "contract_id": "_process_batch", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 328, - "end_line": 426, + "start_line": 331, + "end_line": 429, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64751,14 +65760,14 @@ } ], "anchor_syntax": "region", - "body": " # #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]\n # @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.\n # @PRE job and batch_rows are valid.\n # @POST TranslationBatch and TranslationRecord rows are created.\n # @SIDE_EFFECT LLM API call; DB writes.\n def _process_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_index: int,\n batch_rows: List[Dict[str, Any]],\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._process_batch\"):\n batch_start = time.monotonic()\n\n # Create batch record\n batch = TranslationBatch(\n id=str(uuid.uuid4()),\n run_id=run_id,\n batch_index=batch_index,\n status=\"RUNNING\",\n total_records=len(batch_rows),\n started_at=datetime.now(timezone.utc),\n )\n self.db.add(batch)\n self.db.flush()\n batch_id = batch.id\n\n result = {\"successful\": 0, \"failed\": 0, \"skipped\": 0, \"retries\": 0}\n\n # Extract source texts for dict filtering\n source_texts = [\n row.get(\"source_text\", \"\")\n for row in batch_rows\n if row.get(\"source_text\")\n ]\n\n # Filter dictionary entries\n dict_matches = DictionaryManager.filter_for_batch(\n self.db, source_texts, job.id\n )\n\n # For each row, determine if we need LLM translation or can use approved translation\n rows_for_llm = []\n pre_translated = []\n\n for row in batch_rows:\n if row.get(\"approved_translation\"):\n pre_translated.append(row)\n else:\n rows_for_llm.append(row)\n\n # Handle pre-translated (approved) rows\n for row in pre_translated:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=row.get(\"approved_translation\"),\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n result[\"successful\"] += 1\n\n # Process rows needing LLM translation\n if rows_for_llm:\n llm_result = self._call_llm_for_batch(\n job=job,\n run_id=run_id,\n batch_rows=rows_for_llm,\n dict_matches=dict_matches,\n batch_id=batch_id,\n )\n result[\"successful\"] += llm_result[\"successful\"]\n result[\"failed\"] += llm_result[\"failed\"]\n result[\"skipped\"] += llm_result[\"skipped\"]\n result[\"retries\"] += llm_result.get(\"retries\", 0)\n\n # Update batch status\n batch.successful_records = result[\"successful\"]\n batch.failed_records = result[\"failed\"]\n batch.completed_at = datetime.now(timezone.utc)\n batch.status = \"COMPLETED\" if result[\"failed\"] == 0 else \"COMPLETED_WITH_ERRORS\"\n self.db.flush()\n\n batch_latency = int((time.monotonic() - batch_start) * 1000)\n logger.reason(f\"Batch {batch_index} complete\", {\n \"batch_id\": batch_id,\n \"latency_ms\": batch_latency,\n **result,\n })\n\n return result\n # #endregion _process_batch\n" + "body": " # #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process]\n # @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records.\n # @PRE job and batch_rows are valid.\n # @POST TranslationBatch and TranslationRecord rows are created.\n # @SIDE_EFFECT LLM API call; DB writes.\n def _process_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_index: int,\n batch_rows: List[Dict[str, Any]],\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._process_batch\"):\n batch_start = time.monotonic()\n\n # Create batch record\n batch = TranslationBatch(\n id=str(uuid.uuid4()),\n run_id=run_id,\n batch_index=batch_index,\n status=\"RUNNING\",\n total_records=len(batch_rows),\n started_at=datetime.now(timezone.utc),\n )\n self.db.add(batch)\n self.db.flush()\n batch_id = batch.id\n\n result = {\"successful\": 0, \"failed\": 0, \"skipped\": 0, \"retries\": 0}\n\n # Extract source texts for dict filtering\n source_texts = [\n row.get(\"source_text\", \"\")\n for row in batch_rows\n if row.get(\"source_text\")\n ]\n\n # Filter dictionary entries\n dict_matches = DictionaryManager.filter_for_batch(\n self.db, source_texts, job.id\n )\n\n # For each row, determine if we need LLM translation or can use approved translation\n rows_for_llm = []\n pre_translated = []\n\n for row in batch_rows:\n if row.get(\"approved_translation\"):\n pre_translated.append(row)\n else:\n rows_for_llm.append(row)\n\n # Handle pre-translated (approved) rows\n for row in pre_translated:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=row.get(\"approved_translation\"),\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n result[\"successful\"] += 1\n\n # Process rows needing LLM translation\n if rows_for_llm:\n llm_result = self._call_llm_for_batch(\n job=job,\n run_id=run_id,\n batch_rows=rows_for_llm,\n dict_matches=dict_matches,\n batch_id=batch_id,\n )\n result[\"successful\"] += llm_result[\"successful\"]\n result[\"failed\"] += llm_result[\"failed\"]\n result[\"skipped\"] += llm_result[\"skipped\"]\n result[\"retries\"] += llm_result.get(\"retries\", 0)\n\n # Update batch status\n batch.successful_records = result[\"successful\"]\n batch.failed_records = result[\"failed\"]\n batch.completed_at = datetime.now(timezone.utc)\n batch.status = \"COMPLETED\" if result[\"failed\"] == 0 else \"COMPLETED_WITH_ERRORS\"\n self.db.flush()\n\n batch_latency = int((time.monotonic() - batch_start) * 1000)\n log.reason(f\"Batch {batch_index} complete\", payload={\n \"batch_id\": batch_id,\n \"latency_ms\": batch_latency,\n **result,\n })\n\n return result\n # #endregion _process_batch\n" }, { "contract_id": "_call_llm_for_batch", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 428, - "end_line": 625, + "start_line": 431, + "end_line": 631, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64796,14 +65805,14 @@ } ], "anchor_syntax": "region", - "body": " # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]\n # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.\n # @PRE job has valid provider_id. batch_rows is non-empty.\n # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.\n # @SIDE_EFFECT HTTP call to LLM provider.\n def _call_llm_for_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_rows: List[Dict[str, Any]],\n dict_matches: List[Dict[str, Any]],\n batch_id: str,\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._call_llm_for_batch\"):\n # Build dictionary section\n dictionary_section = \"\"\n if dict_matches:\n glossary_lines = []\n for m in dict_matches:\n glossary_lines.append(\n f\"- '{m['source_term']}' -> '{m['target_term']}'\"\n f\"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}\"\n )\n dictionary_section = (\n \"Terminology dictionary (use these translations when applicable):\\n\"\n + \"\\n\".join(glossary_lines)\n + \"\\n\\n\"\n )\n\n # Build rows JSON for LLM\n rows_json = json.dumps([\n {\n \"row_id\": str(row.get(\"row_index\", idx)),\n \"text\": row.get(\"source_text\", \"\"),\n }\n for idx, row in enumerate(batch_rows)\n ], indent=2)\n\n # Build prompt\n prompt = render_prompt(\n DEFAULT_EXECUTION_PROMPT_TEMPLATE,\n {\n \"source_language\": job.source_dialect or \"SQL\",\n \"target_language\": job.target_language or job.target_dialect or \"en\",\n \"source_dialect\": job.source_dialect or \"\",\n \"target_dialect\": job.target_dialect or \"\",\n \"translation_column\": job.translation_column or \"\",\n \"dictionary_section\": dictionary_section,\n \"rows_json\": rows_json,\n \"row_count\": str(len(batch_rows)),\n },\n )\n\n # Call LLM with retry\n llm_response = None\n last_error = None\n retries = 0\n\n for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):\n try:\n llm_response = self._call_llm(job, prompt)\n break\n except Exception as e:\n last_error = str(e)\n retries += 1\n logger.explore(f\"LLM call failed (attempt {attempt})\", {\n \"batch_id\": batch_id,\n \"error\": last_error,\n \"attempt\": attempt,\n })\n if attempt < MAX_RETRIES_PER_BATCH:\n time.sleep(2 ** attempt) # Exponential backoff\n else:\n logger.explore(\"LLM call exhausted retries\", {\n \"batch_id\": batch_id,\n \"last_error\": last_error,\n })\n\n if llm_response is None:\n # All retries failed — mark all rows as failed\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"FAILED\",\n error_message=f\"LLM call failed after {retries} retries: {last_error}\",\n )\n self.db.add(record)\n return {\"successful\": 0, \"failed\": len(batch_rows), \"skipped\": 0, \"retries\": retries}\n\n # Parse LLM response\n try:\n translations = self._parse_llm_response(llm_response, len(batch_rows))\n except ValueError as e:\n # Parse failure — mark all rows as SKIPPED\n logger.explore(\"LLM response parse failed\", {\n \"batch_id\": batch_id,\n \"error\": str(e),\n \"response_preview\": llm_response[:500] if llm_response else \"\",\n })\n skipped = len(batch_rows)\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=f\"LLM parse failure: {e}\",\n )\n self.db.add(record)\n return {\n \"successful\": 0,\n \"failed\": 0,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n\n successful = 0\n failed = 0\n skipped = 0\n\n for row in batch_rows:\n row_id = str(row.get(\"row_index\", \"\"))\n translation = translations.get(row_id)\n\n if translation is None:\n # NULL translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"NULL translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n if translation.strip() == \"\":\n # Empty translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"Empty translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n successful += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=translation,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n\n return {\n \"successful\": successful,\n \"failed\": failed,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n # #endregion _call_llm_for_batch\n" + "body": " # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch]\n # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response.\n # @PRE job has valid provider_id. batch_rows is non-empty.\n # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.\n # @SIDE_EFFECT HTTP call to LLM provider.\n def _call_llm_for_batch(\n self,\n job: TranslationJob,\n run_id: str,\n batch_rows: List[Dict[str, Any]],\n dict_matches: List[Dict[str, Any]],\n batch_id: str,\n ) -> Dict[str, int]:\n with belief_scope(\"TranslationExecutor._call_llm_for_batch\"):\n # Build dictionary section\n dictionary_section = \"\"\n if dict_matches:\n glossary_lines = []\n for m in dict_matches:\n glossary_lines.append(\n f\"- '{m['source_term']}' -> '{m['target_term']}'\"\n f\"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}\"\n )\n dictionary_section = (\n \"Terminology dictionary (use these translations when applicable):\\n\"\n + \"\\n\".join(glossary_lines)\n + \"\\n\\n\"\n )\n\n # Build rows JSON for LLM\n rows_json = json.dumps([\n {\n \"row_id\": str(row.get(\"row_index\", idx)),\n \"text\": row.get(\"source_text\", \"\"),\n }\n for idx, row in enumerate(batch_rows)\n ], indent=2)\n\n # Build prompt\n prompt = render_prompt(\n DEFAULT_EXECUTION_PROMPT_TEMPLATE,\n {\n \"source_language\": job.source_dialect or \"SQL\",\n \"target_language\": job.target_language or job.target_dialect or \"en\",\n \"source_dialect\": job.source_dialect or \"\",\n \"target_dialect\": job.target_dialect or \"\",\n \"translation_column\": job.translation_column or \"\",\n \"dictionary_section\": dictionary_section,\n \"rows_json\": rows_json,\n \"row_count\": str(len(batch_rows)),\n },\n )\n\n # Call LLM with retry\n llm_response = None\n last_error = None\n retries = 0\n\n for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):\n try:\n llm_response = self._call_llm(job, prompt)\n break\n except Exception as e:\n last_error = str(e)\n retries += 1\n log.explore(\"LLM call failed\", error=\"LLM call failed, retrying\",\n payload={\n \"batch_id\": batch_id,\n \"error\": last_error,\n \"attempt\": attempt,\n })\n if attempt < MAX_RETRIES_PER_BATCH:\n time.sleep(2 ** attempt) # Exponential backoff\n else:\n log.explore(\"LLM call exhausted retries\", error=\"LLM retries exhausted\",\n payload={\n \"batch_id\": batch_id,\n \"last_error\": last_error,\n })\n\n if llm_response is None:\n # All retries failed — mark all rows as failed\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"FAILED\",\n error_message=f\"LLM call failed after {retries} retries: {last_error}\",\n )\n self.db.add(record)\n return {\"successful\": 0, \"failed\": len(batch_rows), \"skipped\": 0, \"retries\": retries}\n\n # Parse LLM response\n try:\n translations = self._parse_llm_response(llm_response, len(batch_rows))\n except ValueError as e:\n # Parse failure — mark all rows as SKIPPED\n log.explore(\"LLM response parse failed\", error=\"Failed to parse LLM JSON response\",\n payload={\n \"batch_id\": batch_id,\n \"error\": str(e),\n \"response_preview\": llm_response[:500] if llm_response else \"\",\n })\n skipped = len(batch_rows)\n for row in batch_rows:\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=None,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=f\"LLM parse failure: {e}\",\n )\n self.db.add(record)\n return {\n \"successful\": 0,\n \"failed\": 0,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n\n successful = 0\n failed = 0\n skipped = 0\n\n for row in batch_rows:\n row_id = str(row.get(\"row_index\", \"\"))\n translation = translations.get(row_id)\n\n if translation is None:\n # NULL translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"NULL translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n if translation.strip() == \"\":\n # Empty translation — skip\n skipped += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=\"\",\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SKIPPED\",\n error_message=\"Empty translation returned by LLM\",\n )\n self.db.add(record)\n continue\n\n successful += 1\n record = TranslationRecord(\n id=str(uuid.uuid4()),\n batch_id=batch_id,\n run_id=run_id,\n source_sql=row.get(\"source_text\", \"\"),\n target_sql=translation,\n source_object_type=\"table_row\",\n source_object_id=row.get(\"row_index\"),\n source_object_name=row.get(\"source_object_name\", \"\"),\n source_data=row.get(\"source_data\", {}),\n status=\"SUCCESS\",\n )\n self.db.add(record)\n\n return {\n \"successful\": successful,\n \"failed\": failed,\n \"skipped\": skipped,\n \"retries\": retries,\n }\n # #endregion _call_llm_for_batch\n" }, { "contract_id": "_call_llm", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 627, - "end_line": 659, + "start_line": 633, + "end_line": 665, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64847,8 +65856,8 @@ "contract_id": "_call_openai_compatible", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 661, - "end_line": 733, + "start_line": 667, + "end_line": 740, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -64886,14 +65895,14 @@ } ], "anchor_syntax": "region", - "body": " # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]\n # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.\n # @PRE Valid API endpoint, key, model, and prompt.\n # @POST Returns response text.\n # @SIDE_EFFECT HTTP POST to LLM API.\n @staticmethod\n def _call_openai_compatible(\n base_url: str,\n api_key: str,\n model: str,\n prompt: str,\n provider_type: str = \"openai\",\n ) -> str:\n with belief_scope(\"TranslationExecutor._call_openai_compatible\"):\n import requests as http_requests\n\n url = f\"{base_url.rstrip('/')}/chat/completions\"\n headers = {\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n }\n payload = {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a database content translation assistant. Translate the provided text accurately, preserving data semantics.\"},\n {\"role\": \"user\", \"content\": prompt},\n ],\n \"temperature\": 0.1,\n \"max_tokens\": 4096,\n }\n # Structured output (response_format) only for native OpenAI — upstream providers routed via\n # Kilo/OpenRouter may not support it (e.g. StepFun returns \"structured_outputs is not supported\")\n if provider_type in (\"openai\", \"openai_compatible\"):\n payload[\"response_format\"] = {\"type\": \"json_object\"}\n\n logger.reason(\n f\"LLM request model={payload.get('model')} \"\n f\"provider_type={provider_type} \"\n f\"response_format={'yes' if 'response_format' in payload else 'no'} \"\n f\"prompt_len={len(prompt)}\"\n )\n response = http_requests.post(url, headers=headers, json=payload, timeout=180)\n if not response.ok:\n logger.explore(\n f\"LLM API error status={response.status_code} \"\n f\"model={payload.get('model')} \"\n f\"body={response.text[:2000]}\"\n )\n response.raise_for_status()\n data = response.json()\n\n choices = data.get(\"choices\", [])\n if not choices:\n raise ValueError(\"LLM returned no choices\")\n\n content = choices[0].get(\"message\", {}).get(\"content\", \"\")\n if not content:\n # Log full response for diagnostics\n finish_reason = choices[0].get(\"finish_reason\", \"unknown\")\n logger.explore(\"LLM returned empty content\", {\n \"finish_reason\": finish_reason,\n \"model\": payload.get(\"model\"),\n \"prompt_len\": len(prompt),\n \"response_keys\": list(data.keys()),\n \"choices_count\": len(choices),\n })\n raise ValueError(\n f\"LLM returned empty content (finish_reason={finish_reason}, \"\n f\"model={payload.get('model')}, prompt_len={len(prompt)})\"\n )\n\n return content\n # #endregion _call_openai_compatible\n" + "body": " # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai]\n # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support.\n # @PRE Valid API endpoint, key, model, and prompt.\n # @POST Returns response text.\n # @SIDE_EFFECT HTTP POST to LLM API.\n @staticmethod\n def _call_openai_compatible(\n base_url: str,\n api_key: str,\n model: str,\n prompt: str,\n provider_type: str = \"openai\",\n ) -> str:\n with belief_scope(\"TranslationExecutor._call_openai_compatible\"):\n import requests as http_requests\n\n url = f\"{base_url.rstrip('/')}/chat/completions\"\n headers = {\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n }\n payload = {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a database content translation assistant. Translate the provided text accurately, preserving data semantics.\"},\n {\"role\": \"user\", \"content\": prompt},\n ],\n \"temperature\": 0.1,\n \"max_tokens\": 4096,\n }\n # Structured output (response_format) only for native OpenAI — upstream providers routed via\n # Kilo/OpenRouter may not support it (e.g. StepFun returns \"structured_outputs is not supported\")\n if provider_type in (\"openai\", \"openai_compatible\"):\n payload[\"response_format\"] = {\"type\": \"json_object\"}\n\n log.reason(\n f\"LLM request model={payload.get('model')} \"\n f\"provider_type={provider_type} \"\n f\"response_format={'yes' if 'response_format' in payload else 'no'} \"\n f\"prompt_len={len(prompt)}\"\n )\n response = http_requests.post(url, headers=headers, json=payload, timeout=180)\n if not response.ok:\n log.explore(\"LLM API error\", error=f\"LLM API returned status {response.status_code}\", payload={\n \"status_code\": response.status_code,\n \"model\": payload.get('model'),\n \"body\": response.text[:2000],\n })\n response.raise_for_status()\n data = response.json()\n\n choices = data.get(\"choices\", [])\n if not choices:\n raise ValueError(\"LLM returned no choices\")\n\n content = choices[0].get(\"message\", {}).get(\"content\", \"\")\n if not content:\n # Log full response for diagnostics\n finish_reason = choices[0].get(\"finish_reason\", \"unknown\")\n log.explore(\"LLM returned empty content\", error=\"Empty response from LLM\",\n payload={\n \"finish_reason\": finish_reason,\n \"model\": payload.get(\"model\"),\n \"prompt_len\": len(prompt),\n \"response_keys\": list(data.keys()),\n \"choices_count\": len(choices),\n })\n raise ValueError(\n f\"LLM returned empty content (finish_reason={finish_reason}, \"\n f\"model={payload.get('model')}, prompt_len={len(prompt)})\"\n )\n\n return content\n # #endregion _call_openai_compatible\n" }, { "contract_id": "_parse_llm_response", "contract_type": "Function", "file_path": "backend/src/plugins/translate/executor.py", - "start_line": 735, - "end_line": 770, + "start_line": 742, + "end_line": 777, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -65102,14 +66111,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]\n# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @RELATION DEPENDS_ON -> [TranslationExecutor]\n# @RELATION DEPENDS_ON -> [SQLGenerator]\n# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @RELATION DEPENDS_ON -> [TranslationPreviewSession]\n# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.\n# @POST Translation run is executed, SQL generated and submitted, events recorded.\n# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.\n# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]\n# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.\n# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.\n# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.\n# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.\n\nimport json\nimport time\nimport uuid\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional, Callable, Tuple\n\nfrom sqlalchemy.orm import Session\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.config_manager import ConfigManager\nfrom ...models.translate import (\n TranslationJob,\n TranslationRun,\n TranslationBatch,\n TranslationRecord,\n TranslationPreviewSession,\n)\nfrom ...schemas.translate import TranslationRunResponse\nfrom .executor import TranslationExecutor\nfrom .sql_generator import SQLGenerator\nfrom .superset_executor import SupersetSqlLabExecutor\nfrom .events import TranslationEventLog\nfrom ..translate.service import TranslateJobService\n\n\n# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]\n# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.\n# @PRE DB session and config manager are available.\n# @POST Runs are created, executed, and finalized with event records.\n# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.\n# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.\n# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]\nclass TranslationOrchestrator:\n\n def __init__(\n self,\n db: Session,\n config_manager: ConfigManager,\n current_user: Optional[str] = None,\n ):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n self.event_log = TranslationEventLog(db)\n self._job: Optional[TranslationJob] = None\n\n # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]\n # @BRIEF Start a new translation run for a job with config snapshot and hash computation.\n # @PRE job_id exists. For manual runs, there must be an accepted preview session.\n # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.\n # @SIDE_EFFECT DB writes; event logging.\n def start_run(\n self,\n job_id: str,\n is_scheduled: bool = False,\n trigger_type: Optional[str] = None,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.start_run\"):\n # Load and validate job\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n self._job = job\n\n logger.reason(\"Starting translation run\", {\n \"job_id\": job_id,\n \"is_scheduled\": is_scheduled,\n })\n\n # Validate preconditions\n self._validate_preconditions(job, is_scheduled=is_scheduled)\n\n # Compute hashes\n config_hash = self._compute_config_hash(job)\n dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)\n\n # Build config snapshot\n config_snapshot = {\n \"source_dialect\": job.source_dialect,\n \"target_dialect\": job.target_dialect,\n \"database_dialect\": job.database_dialect,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n \"target_schema\": job.target_schema,\n \"target_table\": job.target_table,\n \"source_key_cols\": job.source_key_cols,\n \"target_key_cols\": job.target_key_cols,\n \"translation_column\": job.translation_column,\n \"context_columns\": job.context_columns,\n \"target_language\": job.target_language,\n \"provider_id\": job.provider_id,\n \"batch_size\": job.batch_size,\n \"upsert_strategy\": job.upsert_strategy,\n \"dictionary_ids\": self._compute_dict_snapshot_hash(job_id),\n }\n\n # Compute key_hash from source_key_cols\n import hashlib\n key_hash_input = json.dumps({\n \"source_key_cols\": job.source_key_cols,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n }, sort_keys=True)\n key_hash = hashlib.sha256(key_hash_input.encode()).hexdigest()[:16]\n\n # Create run record with all hash/snapshot fields\n run = TranslationRun(\n id=str(uuid.uuid4()),\n job_id=job_id,\n status=\"PENDING\",\n trigger_type=trigger_type or (\"scheduled\" if is_scheduled else \"manual\"),\n config_snapshot=config_snapshot,\n key_hash=key_hash,\n config_hash=config_hash,\n dict_snapshot_hash=dict_snapshot_hash,\n created_by=self.current_user,\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(run)\n self.db.flush()\n\n # Record event\n self.event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\n \"is_scheduled\": is_scheduled,\n \"trigger_type\": run.trigger_type,\n \"config_hash\": config_hash,\n \"dict_snapshot_hash\": dict_snapshot_hash,\n \"key_hash\": key_hash,\n },\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n logger.reflect(\"Run created\", {\n \"run_id\": run.id,\n \"job_id\": job_id,\n \"status\": run.status,\n \"trigger_type\": run.trigger_type,\n })\n return run\n # #endregion start_run\n\n # #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]\n # @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.\n # @PRE run is in PENDING status.\n # @POST Run is executed, SQL generated, Superset submission attempted.\n # @SIDE_EFFECT LLM calls, DB writes, Superset API calls.\n def execute_run(\n self,\n run: TranslationRun,\n on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,\n skip_insert: bool = False,\n full_translation: bool = False,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.execute_run\"):\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n if run.status != \"PENDING\":\n raise ValueError(\n f\"Cannot execute run in status '{run.status}'. \"\n f\"Run must be in PENDING status.\"\n )\n\n logger.reason(\"Executing run\", {\n \"run_id\": run.id,\n \"job_id\": job.id,\n \"skip_insert\": skip_insert,\n })\n\n # Record translation phase start\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"TRANSLATION_PHASE_STARTED\",\n payload={},\n created_by=self.current_user,\n )\n\n # Dispatch executor\n executor = TranslationExecutor(\n self.db, self.config_manager, self.current_user,\n on_batch_progress=on_batch_progress,\n )\n try:\n run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)\n except Exception as e:\n logger.explore(\"Translation execution failed\", {\n \"run_id\": run.id,\n \"error\": str(e),\n })\n run.status = \"FAILED\"\n run.error_message = f\"Translation execution failed: {e}\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_FAILED\",\n payload={\"error\": str(e), \"phase\": \"translation\"},\n created_by=self.current_user,\n )\n self.db.commit()\n return run\n\n # Record translation phase complete\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"TRANSLATION_PHASE_COMPLETED\",\n payload={\n \"total\": run.total_records,\n \"successful\": run.successful_records,\n \"failed\": run.failed_records,\n \"skipped\": run.skipped_records,\n },\n created_by=self.current_user,\n )\n\n # Skip insert phase if requested (e.g., for preview-only execution)\n if skip_insert:\n run.status = \"COMPLETED\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_COMPLETED\",\n payload={\"skip_insert\": True},\n created_by=self.current_user,\n )\n self.db.commit()\n return run\n\n # Generate SQL and submit to Superset\n insert_result = self._generate_and_insert_sql(job, run)\n run.insert_status = insert_result.get(\"status\")\n run.superset_execution_id = str(insert_result.get(\"query_id\") or \"\")\n run.superset_execution_log = insert_result\n run.status = \"COMPLETED\" if insert_result.get(\"status\") == \"success\" else \"COMPLETED\"\n if insert_result.get(\"error_message\"):\n run.error_message = insert_result[\"error_message\"]\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n # Record terminal event\n terminal_event = \"RUN_COMPLETED\"\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=terminal_event,\n payload={\n \"insert_status\": insert_result.get(\"status\"),\n \"query_id\": insert_result.get(\"query_id\"),\n \"rows_affected\": insert_result.get(\"rows_affected\"),\n \"total_records\": run.total_records,\n \"successful\": run.successful_records,\n \"failed\": run.failed_records,\n \"skipped\": run.skipped_records,\n },\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n logger.reflect(\"Run execution complete\", {\n \"run_id\": run.id,\n \"status\": run.status,\n \"insert_status\": run.insert_status,\n })\n return run\n # #endregion execute_run\n\n # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]\n # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.\n # @PRE job has target table configured. run has successful records.\n # @POST SQL is generated and submitted. Returns execution result.\n # @SIDE_EFFECT Superset API call; event logging.\n def _generate_and_insert_sql(\n self,\n job: TranslationJob,\n run: TranslationRun,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator._generate_and_insert_sql\"):\n # Fetch successful records\n records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run.id,\n TranslationRecord.status == \"SUCCESS\",\n TranslationRecord.target_sql.isnot(None),\n )\n .all()\n )\n\n if not records:\n logger.reason(\"No successful records to insert\", {\"run_id\": run.id})\n return {\"status\": \"skipped\", \"reason\": \"no_records\", \"query_id\": None}\n\n logger.reason(f\"Generating SQL for {len(records)} records\", {\n \"run_id\": run.id,\n \"dialect\": job.database_dialect or job.target_dialect,\n })\n\n # Build rows for SQL generation\n # Only include the translation column — context columns are for LLM only\n columns = []\n if job.translation_column:\n columns.append(job.translation_column)\n\n # Include key columns if present (for UPSERT matching)\n if job.target_key_cols:\n for k in job.target_key_cols:\n if k not in columns:\n columns.append(k)\n\n rows_for_sql = []\n for rec in records:\n row_data = {}\n if job.translation_column:\n row_data[job.translation_column] = rec.target_sql or \"\"\n if job.target_key_cols:\n source_data = rec.source_data or {}\n for k in job.target_key_cols:\n # Use source_data value if available, fall back to empty string\n row_data[k] = source_data.get(k, \"\")\n rows_for_sql.append(row_data)\n\n if not columns:\n # Use target_sql as the sole column\n columns = [job.translation_column or \"translated_text\"]\n rows_for_sql = [{columns[0]: rec.target_sql or \"\"} for rec in records]\n\n # Generate SQL\n try:\n sql, row_count = SQLGenerator.generate(\n dialect=job.database_dialect or job.target_dialect or \"postgresql\",\n target_schema=job.target_schema,\n target_table=job.target_table or \"translated_data\",\n columns=columns,\n rows=rows_for_sql,\n key_columns=job.target_key_cols,\n upsert_strategy=job.upsert_strategy or \"MERGE\",\n )\n except ValueError as e:\n logger.explore(\"SQL generation failed\", {\"error\": str(e)})\n return {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase start\n logger.reason(\"Insert SQL generated\", {\n \"run_id\": run.id,\n \"sql_preview\": sql[:500],\n \"sql_length\": len(sql),\n \"row_count\": row_count,\n \"dialect\": job.database_dialect or job.target_dialect or \"postgresql\",\n \"columns\": columns,\n \"has_key_cols\": bool(job.target_key_cols),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_STARTED\",\n payload={\"sql_length\": len(sql), \"row_count\": row_count},\n created_by=self.current_user,\n )\n\n # Submit to Superset\n try:\n env_id = job.environment_id or job.source_dialect or \"\"\n # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment\n target_db_id = None\n if job.target_database_id:\n try:\n target_db_id = int(job.target_database_id)\n except (ValueError, TypeError):\n # Could be a UUID — pass as-is, executor will resolve it\n target_db_id = job.target_database_id\n logger.reason(\"target_database_id is not an integer, will resolve as UUID\", {\n \"target_database_id\": job.target_database_id,\n })\n executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)\n result = executor.execute_and_poll(\n sql=sql,\n max_polls=30,\n poll_interval_seconds=2.0,\n )\n except Exception as e:\n logger.explore(\"Superset SQL submission failed\", {\n \"run_id\": run.id,\n \"error\": str(e),\n })\n result = {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase complete\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_COMPLETED\",\n payload=result,\n created_by=self.current_user,\n )\n\n return result\n # #endregion _generate_and_insert_sql\n\n # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]\n # @BRIEF Validate preconditions before starting a translation run.\n # @PRE None.\n # @POST Raises ValueError if preconditions are not met.\n # @SIDE_EFFECT None.\n def _validate_preconditions(\n self,\n job: TranslationJob,\n is_scheduled: bool = False,\n ) -> None:\n with belief_scope(\"TranslationOrchestrator._validate_preconditions\"):\n # Job must be in valid status\n if job.status in (\"DRAFT\",):\n raise ValueError(\n f\"Cannot run job '{job.id}' in status '{job.status}'. \"\n f\"Job must be READY, ACTIVE, or COMPLETED.\"\n )\n\n # Must have target table configured\n if not job.target_table:\n raise ValueError(\n f\"Job '{job.id}' has no target table configured. \"\n \"Configure a target table before running.\"\n )\n\n # Must have LLM provider configured\n if not job.provider_id:\n raise ValueError(\n f\"Job '{job.id}' has no LLM provider configured. \"\n \"Select an LLM provider before running.\"\n )\n\n # Must have a translation column\n if not job.translation_column:\n raise ValueError(\n f\"Job '{job.id}' has no translation column configured. \"\n \"Select a translation column before running.\"\n )\n\n # For manual runs, must have accepted preview\n if not is_scheduled:\n accepted_session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job.id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not accepted_session:\n raise ValueError(\n f\"Job '{job.id}' has no accepted preview session. \"\n \"Run and accept a preview before executing a manual translation run.\"\n )\n\n logger.reason(\"Preconditions validated\", {\n \"job_id\": job.id,\n \"is_scheduled\": is_scheduled,\n })\n # #endregion _validate_preconditions\n\n # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]\n # @BRIEF Retry failed batches in a run by re-processing failed records.\n # @PRE run exists and has failed batches.\n # @POST Failed batches are re-processed; run status and stats updated.\n # @SIDE_EFFECT LLM calls; DB writes; event logging.\n def retry_failed_batches(\n self,\n run_id: str,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.retry_failed_batches\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n # Find failed batches\n failed_batches = (\n self.db.query(TranslationBatch)\n .filter(\n TranslationBatch.run_id == run_id,\n TranslationBatch.status.in_([\"FAILED\", \"COMPLETED_WITH_ERRORS\"]),\n )\n .all()\n )\n\n if not failed_batches:\n raise ValueError(f\"No failed batches found for run '{run_id}'\")\n\n logger.reason(\"Retrying failed batches\", {\n \"run_id\": run_id,\n \"batch_count\": len(failed_batches),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_RETRYING\",\n payload={\"batch_count\": len(failed_batches)},\n created_by=self.current_user,\n )\n\n # Re-process each failed batch\n executor = TranslationExecutor(self.db, self.config_manager, self.current_user)\n run.status = \"RUNNING\"\n self.db.flush()\n\n for batch in failed_batches:\n # Fetch failed records for this batch\n failed_records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.batch_id == batch.id,\n TranslationRecord.status == \"FAILED\",\n )\n .all()\n )\n\n if not failed_records:\n continue\n\n # Build rows from failed records\n retry_rows = []\n for rec in failed_records:\n retry_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": None,\n \"source_object_name\": rec.source_object_name or \"\",\n })\n\n # Process retry batch\n result = executor._process_batch(\n job=job,\n run_id=run_id,\n batch_index=batch.batch_index,\n batch_rows=retry_rows,\n )\n\n # Update run stats\n run.successful_records = (run.successful_records or 0) + result[\"successful\"]\n run.failed_records = (run.failed_records or 0) + result[\"failed\"]\n run.skipped_records = (run.skipped_records or 0) + result[\"skipped\"]\n self.db.flush()\n\n # Update run status\n if run.failed_records == 0:\n run.status = \"COMPLETED\"\n elif run.successful_records == 0:\n run.status = \"FAILED\"\n else:\n run.status = \"COMPLETED\"\n\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_COMPLETED\",\n payload={\"retry\": True},\n created_by=self.current_user,\n )\n\n self.db.commit()\n logger.reflect(\"Retry complete\", {\n \"run_id\": run_id,\n \"status\": run.status,\n })\n return run\n # #endregion retry_failed_batches\n\n # #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]\n # @BRIEF Retry the SQL insert phase for a completed run.\n # @PRE run exists and has successful records.\n # @POST SQL is regenerated and re-submitted to Superset.\n # @SIDE_EFFECT Superset API call; event logging.\n def retry_insert(\n self,\n run_id: str,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.retry_insert\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n logger.reason(\"Retrying insert phase\", {\n \"run_id\": run_id,\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_RETRY_INSERT\",\n payload={},\n created_by=self.current_user,\n )\n\n # Regenerate SQL and submit\n insert_result = self._generate_and_insert_sql(job, run)\n\n # Update run\n run.insert_status = insert_result.get(\"status\")\n run.superset_execution_id = str(insert_result.get(\"query_id\") or \"\")\n if insert_result.get(\"error_message\"):\n run.error_message = insert_result[\"error_message\"]\n self.db.flush()\n self.db.commit()\n self.db.refresh(run)\n\n logger.reflect(\"Insert retry complete\", {\n \"run_id\": run_id,\n \"insert_status\": run.insert_status,\n })\n return run\n # #endregion retry_insert\n\n # #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]\n # @BRIEF Cancel a running translation run.\n # @PRE run is in PENDING or RUNNING status.\n # @POST Run status is set to CANCELLED; event recorded.\n # @SIDE_EFFECT DB write; records event.\n def cancel_run(self, run_id: str) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.cancel_run\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n if run.status not in (\"PENDING\", \"RUNNING\"):\n raise ValueError(\n f\"Cannot cancel run in status '{run.status}'. \"\n f\"Only PENDING or RUNNING runs can be cancelled.\"\n )\n\n run.status = \"CANCELLED\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n # Record terminal event directly — we are the source of the terminal event\n self.event_log._create_event_raw(\n job_id=run.job_id,\n run_id=run.id,\n event_type=\"RUN_CANCELLED\",\n payload={},\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n logger.reflect(\"Run cancelled\", {\n \"run_id\": run_id,\n })\n return run\n # #endregion cancel_run\n\n # #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]\n # @BRIEF Get run status with statistics and event invariant checks.\n # @PRE run_id exists.\n # @POST Returns dict with run details.\n def get_run_status(self, run_id: str) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator.get_run_status\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n # Count batches\n batch_count = (\n self.db.query(TranslationBatch)\n .filter(TranslationBatch.run_id == run_id)\n .count()\n )\n\n # Get event invariants (lightweight — only fetches event_type column)\n invariants = self.event_log.get_run_event_invariants_lightweight(run_id)\n\n return {\n \"id\": run.id,\n \"job_id\": run.job_id,\n \"status\": run.status,\n \"started_at\": run.started_at.isoformat() if run.started_at else None,\n \"completed_at\": run.completed_at.isoformat() if run.completed_at else None,\n \"error_message\": run.error_message,\n \"total_records\": run.total_records or 0,\n \"successful_records\": run.successful_records or 0,\n \"failed_records\": run.failed_records or 0,\n \"skipped_records\": run.skipped_records or 0,\n \"insert_status\": run.insert_status,\n \"superset_execution_id\": run.superset_execution_id,\n \"batch_count\": batch_count,\n \"event_invariants\": invariants,\n \"created_by\": run.created_by,\n \"created_at\": run.created_at.isoformat() if run.created_at else None,\n }\n # #endregion get_run_status\n\n # #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]\n # @BRIEF Get paginated records for a run with optional status filter.\n # @PRE run_id exists.\n # @POST Returns dict with records and pagination info.\n def get_run_records(\n self,\n run_id: str,\n page: int = 1,\n page_size: int = 50,\n status_filter: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator.get_run_records\"):\n query = self.db.query(TranslationRecord).filter(\n TranslationRecord.run_id == run_id\n )\n\n if status_filter:\n query = query.filter(TranslationRecord.status == status_filter)\n\n total = query.count()\n offset = (page - 1) * page_size\n records = (\n query.order_by(TranslationRecord.created_at.desc())\n .offset(offset)\n .limit(page_size)\n .all()\n )\n\n return {\n \"items\": [\n {\n \"id\": r.id,\n \"batch_id\": r.batch_id,\n \"source_sql\": r.source_sql,\n \"target_sql\": r.target_sql,\n \"source_object_type\": r.source_object_type,\n \"source_object_id\": r.source_object_id,\n \"source_object_name\": r.source_object_name,\n \"status\": r.status,\n \"error_message\": r.error_message,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n }\n for r in records\n ],\n \"total\": total,\n \"page\": page,\n \"page_size\": page_size,\n \"status_filter\": status_filter,\n }\n # #endregion get_run_records\n\n # #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]\n # @BRIEF Get paginated run history for a job.\n # @PRE job_id exists.\n # @POST Returns tuple of (total_count, list_of_runs).\n def get_run_history(\n self,\n job_id: str,\n page: int = 1,\n page_size: int = 20,\n ) -> Tuple[int, List[Dict[str, Any]]]:\n with belief_scope(\"TranslationOrchestrator.get_run_history\"):\n query = self.db.query(TranslationRun).filter(\n TranslationRun.job_id == job_id\n )\n total = query.count()\n runs = (\n query.order_by(TranslationRun.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n\n return total, [\n {\n \"id\": r.id,\n \"job_id\": r.job_id,\n \"status\": r.status,\n \"started_at\": r.started_at.isoformat() if r.started_at else None,\n \"completed_at\": r.completed_at.isoformat() if r.completed_at else None,\n \"error_message\": r.error_message,\n \"total_records\": r.total_records or 0,\n \"successful_records\": r.successful_records or 0,\n \"failed_records\": r.failed_records or 0,\n \"skipped_records\": r.skipped_records or 0,\n \"insert_status\": r.insert_status,\n \"superset_execution_id\": r.superset_execution_id,\n \"created_by\": r.created_by,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n }\n for r in runs\n ]\n # #endregion get_run_history\n\n # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]\n # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.\n @staticmethod\n def _compute_config_hash(job: TranslationJob) -> str:\n import hashlib\n config_str = json.dumps({\n \"source_dialect\": job.source_dialect,\n \"target_dialect\": job.target_dialect,\n \"source_datasource_id\": job.source_datasource_id,\n \"translation_column\": job.translation_column,\n \"context_columns\": job.context_columns,\n \"target_language\": job.target_language,\n \"provider_id\": job.provider_id,\n \"batch_size\": job.batch_size,\n \"upsert_strategy\": job.upsert_strategy,\n }, sort_keys=True)\n return hashlib.sha256(config_str.encode()).hexdigest()[:16]\n # #endregion _compute_config_hash\n\n # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]\n # @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.\n def _compute_dict_snapshot_hash(self, job_id: str) -> str:\n import hashlib\n from ...models.translate import TranslationJobDictionary\n dict_links = (\n self.db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n dict_ids = sorted([dl.dictionary_id for dl in dict_links])\n hash_input = \",\".join(dict_ids)\n return hashlib.sha256(hash_input.encode()).hexdigest()[:16]\n # #endregion _compute_dict_snapshot_hash\n\n\n# #endregion TranslationOrchestrator\n# #endregion TranslationOrchestrator\n" + "body": "# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run]\n# @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @RELATION DEPENDS_ON -> [TranslationExecutor]\n# @RELATION DEPENDS_ON -> [SQLGenerator]\n# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @RELATION DEPENDS_ON -> [TranslationPreviewSession]\n# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable.\n# @POST Translation run is executed, SQL generated and submitted, events recorded.\n# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events.\n# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]\n# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run.\n# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary.\n# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale.\n# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant.\n\nimport json\nimport time\nimport uuid\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional, Callable, Tuple\n\nfrom sqlalchemy.orm import Session\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.config_manager import ConfigManager\nfrom ...models.translate import (\n TranslationJob,\n TranslationRun,\n TranslationBatch,\n TranslationRecord,\n TranslationPreviewSession,\n)\nfrom ...schemas.translate import TranslationRunResponse\nfrom .executor import TranslationExecutor\nfrom .sql_generator import SQLGenerator\nfrom .superset_executor import SupersetSqlLabExecutor\nfrom .events import TranslationEventLog\nfrom ..translate.service import TranslateJobService\n\nlog = MarkerLogger(\"TranslationOrchestrator\")\n\n# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator]\n# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.\n# @PRE DB session and config manager are available.\n# @POST Runs are created, executed, and finalized with event records.\n# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows.\n# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED.\n# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun]\nclass TranslationOrchestrator:\n\n def __init__(\n self,\n db: Session,\n config_manager: ConfigManager,\n current_user: Optional[str] = None,\n ):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n self.event_log = TranslationEventLog(db)\n self._job: Optional[TranslationJob] = None\n\n # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]\n # @BRIEF Start a new translation run for a job with config snapshot and hash computation.\n # @PRE job_id exists. For manual runs, there must be an accepted preview session.\n # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.\n # @SIDE_EFFECT DB writes; event logging.\n def start_run(\n self,\n job_id: str,\n is_scheduled: bool = False,\n trigger_type: Optional[str] = None,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.start_run\"):\n # Load and validate job\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n self._job = job\n\n log.reason(\"Starting translation run\", payload={\n \"job_id\": job_id,\n \"is_scheduled\": is_scheduled,\n })\n\n # Validate preconditions\n self._validate_preconditions(job, is_scheduled=is_scheduled)\n\n # Compute hashes\n config_hash = self._compute_config_hash(job)\n dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)\n\n # Build config snapshot\n config_snapshot = {\n \"source_dialect\": job.source_dialect,\n \"target_dialect\": job.target_dialect,\n \"database_dialect\": job.database_dialect,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n \"target_schema\": job.target_schema,\n \"target_table\": job.target_table,\n \"source_key_cols\": job.source_key_cols,\n \"target_key_cols\": job.target_key_cols,\n \"translation_column\": job.translation_column,\n \"context_columns\": job.context_columns,\n \"target_language\": job.target_language,\n \"provider_id\": job.provider_id,\n \"batch_size\": job.batch_size,\n \"upsert_strategy\": job.upsert_strategy,\n \"dictionary_ids\": self._compute_dict_snapshot_hash(job_id),\n }\n\n # Compute key_hash from source_key_cols\n import hashlib\n key_hash_input = json.dumps({\n \"source_key_cols\": job.source_key_cols,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n }, sort_keys=True)\n key_hash = hashlib.sha256(key_hash_input.encode()).hexdigest()[:16]\n\n # Create run record with all hash/snapshot fields\n run = TranslationRun(\n id=str(uuid.uuid4()),\n job_id=job_id,\n status=\"PENDING\",\n trigger_type=trigger_type or (\"scheduled\" if is_scheduled else \"manual\"),\n config_snapshot=config_snapshot,\n key_hash=key_hash,\n config_hash=config_hash,\n dict_snapshot_hash=dict_snapshot_hash,\n created_by=self.current_user,\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(run)\n self.db.flush()\n\n # Record event\n self.event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\n \"is_scheduled\": is_scheduled,\n \"trigger_type\": run.trigger_type,\n \"config_hash\": config_hash,\n \"dict_snapshot_hash\": dict_snapshot_hash,\n \"key_hash\": key_hash,\n },\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n log.reflect(\"Run created\", payload={\n \"run_id\": run.id,\n \"job_id\": job_id,\n \"status\": run.status,\n \"trigger_type\": run.trigger_type,\n })\n return run\n # #endregion start_run\n\n # #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute]\n # @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset.\n # @PRE run is in PENDING status.\n # @POST Run is executed, SQL generated, Superset submission attempted.\n # @SIDE_EFFECT LLM calls, DB writes, Superset API calls.\n def execute_run(\n self,\n run: TranslationRun,\n on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,\n skip_insert: bool = False,\n full_translation: bool = False,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.execute_run\"):\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n if run.status != \"PENDING\":\n raise ValueError(\n f\"Cannot execute run in status '{run.status}'. \"\n f\"Run must be in PENDING status.\"\n )\n\n log.reason(\"Executing run\", payload={\n \"run_id\": run.id,\n \"job_id\": job.id,\n \"skip_insert\": skip_insert,\n })\n\n # Record translation phase start\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"TRANSLATION_PHASE_STARTED\",\n payload={},\n created_by=self.current_user,\n )\n\n # Dispatch executor\n executor = TranslationExecutor(\n self.db, self.config_manager, self.current_user,\n on_batch_progress=on_batch_progress,\n )\n try:\n run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation)\n except Exception as e:\n log.explore(\"Translation execution failed\", error=str(e), payload={\n \"run_id\": run.id,\n })\n run.status = \"FAILED\"\n run.error_message = f\"Translation execution failed: {e}\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_FAILED\",\n payload={\"error\": str(e), \"phase\": \"translation\"},\n created_by=self.current_user,\n )\n self.db.commit()\n return run\n\n # Record translation phase complete\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"TRANSLATION_PHASE_COMPLETED\",\n payload={\n \"total\": run.total_records,\n \"successful\": run.successful_records,\n \"failed\": run.failed_records,\n \"skipped\": run.skipped_records,\n },\n created_by=self.current_user,\n )\n\n # Skip insert phase if requested (e.g., for preview-only execution)\n if skip_insert:\n run.status = \"COMPLETED\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_COMPLETED\",\n payload={\"skip_insert\": True},\n created_by=self.current_user,\n )\n self.db.commit()\n return run\n\n # Generate SQL and submit to Superset\n insert_result = self._generate_and_insert_sql(job, run)\n run.insert_status = insert_result.get(\"status\")\n run.superset_execution_id = str(insert_result.get(\"query_id\") or \"\")\n run.superset_execution_log = insert_result\n run.status = \"COMPLETED\" if insert_result.get(\"status\") == \"success\" else \"COMPLETED\"\n if insert_result.get(\"error_message\"):\n run.error_message = insert_result[\"error_message\"]\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n # Record terminal event\n terminal_event = \"RUN_COMPLETED\"\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=terminal_event,\n payload={\n \"insert_status\": insert_result.get(\"status\"),\n \"query_id\": insert_result.get(\"query_id\"),\n \"rows_affected\": insert_result.get(\"rows_affected\"),\n \"total_records\": run.total_records,\n \"successful\": run.successful_records,\n \"failed\": run.failed_records,\n \"skipped\": run.skipped_records,\n },\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n log.reflect(\"Run execution complete\", payload={\n \"run_id\": run.id,\n \"status\": run.status,\n \"insert_status\": run.insert_status,\n })\n return run\n # #endregion execute_run\n\n # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]\n # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.\n # @PRE job has target table configured. run has successful records.\n # @POST SQL is generated and submitted. Returns execution result.\n # @SIDE_EFFECT Superset API call; event logging.\n def _generate_and_insert_sql(\n self,\n job: TranslationJob,\n run: TranslationRun,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator._generate_and_insert_sql\"):\n # Fetch successful records\n records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run.id,\n TranslationRecord.status == \"SUCCESS\",\n TranslationRecord.target_sql.isnot(None),\n )\n .all()\n )\n\n if not records:\n log.reason(\"No successful records to insert\", payload={\"run_id\": run.id})\n return {\"status\": \"skipped\", \"reason\": \"no_records\", \"query_id\": None}\n\n log.reason(f\"Generating SQL for {len(records)} records\", payload={\n \"run_id\": run.id,\n \"dialect\": job.database_dialect or job.target_dialect,\n })\n\n # Build rows for SQL generation\n # Only include the translation column — context columns are for LLM only\n columns = []\n if job.translation_column:\n columns.append(job.translation_column)\n\n # Include key columns if present (for UPSERT matching)\n if job.target_key_cols:\n for k in job.target_key_cols:\n if k not in columns:\n columns.append(k)\n\n rows_for_sql = []\n for rec in records:\n row_data = {}\n if job.translation_column:\n row_data[job.translation_column] = rec.target_sql or \"\"\n if job.target_key_cols:\n source_data = rec.source_data or {}\n for k in job.target_key_cols:\n # Use source_data value if available, fall back to empty string\n row_data[k] = source_data.get(k, \"\")\n rows_for_sql.append(row_data)\n\n if not columns:\n # Use target_sql as the sole column\n columns = [job.translation_column or \"translated_text\"]\n rows_for_sql = [{columns[0]: rec.target_sql or \"\"} for rec in records]\n\n # Generate SQL\n try:\n sql, row_count = SQLGenerator.generate(\n dialect=job.database_dialect or job.target_dialect or \"postgresql\",\n target_schema=job.target_schema,\n target_table=job.target_table or \"translated_data\",\n columns=columns,\n rows=rows_for_sql,\n key_columns=job.target_key_cols,\n upsert_strategy=job.upsert_strategy or \"MERGE\",\n )\n except ValueError as e:\n log.explore(\"SQL generation failed\", error=str(e))\n return {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase start\n log.reason(\"Insert SQL generated\", payload={\n \"run_id\": run.id,\n \"sql_preview\": sql[:500],\n \"sql_length\": len(sql),\n \"row_count\": row_count,\n \"dialect\": job.database_dialect or job.target_dialect or \"postgresql\",\n \"columns\": columns,\n \"has_key_cols\": bool(job.target_key_cols),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_STARTED\",\n payload={\"sql_length\": len(sql), \"row_count\": row_count},\n created_by=self.current_user,\n )\n\n # Submit to Superset\n try:\n env_id = job.environment_id or job.source_dialect or \"\"\n # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment\n target_db_id = None\n if job.target_database_id:\n try:\n target_db_id = int(job.target_database_id)\n except (ValueError, TypeError):\n # Could be a UUID — pass as-is, executor will resolve it\n target_db_id = job.target_database_id\n log.reason(\"target_database_id is not an integer, will resolve as UUID\", payload={\n \"target_database_id\": job.target_database_id,\n })\n executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)\n result = executor.execute_and_poll(\n sql=sql,\n max_polls=30,\n poll_interval_seconds=2.0,\n )\n except Exception as e:\n log.explore(\"Superset SQL submission failed\", error=str(e), payload={\n \"run_id\": run.id,\n })\n result = {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase complete\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_COMPLETED\",\n payload=result,\n created_by=self.current_user,\n )\n\n return result\n # #endregion _generate_and_insert_sql\n\n # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]\n # @BRIEF Validate preconditions before starting a translation run.\n # @PRE None.\n # @POST Raises ValueError if preconditions are not met.\n # @SIDE_EFFECT None.\n def _validate_preconditions(\n self,\n job: TranslationJob,\n is_scheduled: bool = False,\n ) -> None:\n with belief_scope(\"TranslationOrchestrator._validate_preconditions\"):\n # Job must be in valid status\n if job.status in (\"DRAFT\",):\n raise ValueError(\n f\"Cannot run job '{job.id}' in status '{job.status}'. \"\n f\"Job must be READY, ACTIVE, or COMPLETED.\"\n )\n\n # Must have target table configured\n if not job.target_table:\n raise ValueError(\n f\"Job '{job.id}' has no target table configured. \"\n \"Configure a target table before running.\"\n )\n\n # Must have LLM provider configured\n if not job.provider_id:\n raise ValueError(\n f\"Job '{job.id}' has no LLM provider configured. \"\n \"Select an LLM provider before running.\"\n )\n\n # Must have a translation column\n if not job.translation_column:\n raise ValueError(\n f\"Job '{job.id}' has no translation column configured. \"\n \"Select a translation column before running.\"\n )\n\n # For manual runs, must have accepted preview\n if not is_scheduled:\n accepted_session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job.id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not accepted_session:\n raise ValueError(\n f\"Job '{job.id}' has no accepted preview session. \"\n \"Run and accept a preview before executing a manual translation run.\"\n )\n\n log.reason(\"Preconditions validated\", payload={\n \"job_id\": job.id,\n \"is_scheduled\": is_scheduled,\n })\n # #endregion _validate_preconditions\n\n # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]\n # @BRIEF Retry failed batches in a run by re-processing failed records.\n # @PRE run exists and has failed batches.\n # @POST Failed batches are re-processed; run status and stats updated.\n # @SIDE_EFFECT LLM calls; DB writes; event logging.\n def retry_failed_batches(\n self,\n run_id: str,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.retry_failed_batches\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n # Find failed batches\n failed_batches = (\n self.db.query(TranslationBatch)\n .filter(\n TranslationBatch.run_id == run_id,\n TranslationBatch.status.in_([\"FAILED\", \"COMPLETED_WITH_ERRORS\"]),\n )\n .all()\n )\n\n if not failed_batches:\n raise ValueError(f\"No failed batches found for run '{run_id}'\")\n\n log.reason(\"Retrying failed batches\", payload={\n \"run_id\": run_id,\n \"batch_count\": len(failed_batches),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_RETRYING\",\n payload={\"batch_count\": len(failed_batches)},\n created_by=self.current_user,\n )\n\n # Re-process each failed batch\n executor = TranslationExecutor(self.db, self.config_manager, self.current_user)\n run.status = \"RUNNING\"\n self.db.flush()\n\n for batch in failed_batches:\n # Fetch failed records for this batch\n failed_records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.batch_id == batch.id,\n TranslationRecord.status == \"FAILED\",\n )\n .all()\n )\n\n if not failed_records:\n continue\n\n # Build rows from failed records\n retry_rows = []\n for rec in failed_records:\n retry_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": None,\n \"source_object_name\": rec.source_object_name or \"\",\n })\n\n # Process retry batch\n result = executor._process_batch(\n job=job,\n run_id=run_id,\n batch_index=batch.batch_index,\n batch_rows=retry_rows,\n )\n\n # Update run stats\n run.successful_records = (run.successful_records or 0) + result[\"successful\"]\n run.failed_records = (run.failed_records or 0) + result[\"failed\"]\n run.skipped_records = (run.skipped_records or 0) + result[\"skipped\"]\n self.db.flush()\n\n # Update run status\n if run.failed_records == 0:\n run.status = \"COMPLETED\"\n elif run.successful_records == 0:\n run.status = \"FAILED\"\n else:\n run.status = \"COMPLETED\"\n\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_COMPLETED\",\n payload={\"retry\": True},\n created_by=self.current_user,\n )\n\n self.db.commit()\n log.reflect(\"Retry complete\", payload={\n \"run_id\": run_id,\n \"status\": run.status,\n })\n return run\n # #endregion retry_failed_batches\n\n # #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert]\n # @BRIEF Retry the SQL insert phase for a completed run.\n # @PRE run exists and has successful records.\n # @POST SQL is regenerated and re-submitted to Superset.\n # @SIDE_EFFECT Superset API call; event logging.\n def retry_insert(\n self,\n run_id: str,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.retry_insert\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n log.reason(\"Retrying insert phase\", payload={\n \"run_id\": run_id,\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_RETRY_INSERT\",\n payload={},\n created_by=self.current_user,\n )\n\n # Regenerate SQL and submit\n insert_result = self._generate_and_insert_sql(job, run)\n\n # Update run\n run.insert_status = insert_result.get(\"status\")\n run.superset_execution_id = str(insert_result.get(\"query_id\") or \"\")\n if insert_result.get(\"error_message\"):\n run.error_message = insert_result[\"error_message\"]\n self.db.flush()\n self.db.commit()\n self.db.refresh(run)\n\n log.reflect(\"Insert retry complete\", payload={\n \"run_id\": run_id,\n \"insert_status\": run.insert_status,\n })\n return run\n # #endregion retry_insert\n\n # #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel]\n # @BRIEF Cancel a running translation run.\n # @PRE run is in PENDING or RUNNING status.\n # @POST Run status is set to CANCELLED; event recorded.\n # @SIDE_EFFECT DB write; records event.\n def cancel_run(self, run_id: str) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.cancel_run\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n if run.status not in (\"PENDING\", \"RUNNING\"):\n raise ValueError(\n f\"Cannot cancel run in status '{run.status}'. \"\n f\"Only PENDING or RUNNING runs can be cancelled.\"\n )\n\n run.status = \"CANCELLED\"\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n # Record terminal event directly — we are the source of the terminal event\n self.event_log._create_event_raw(\n job_id=run.job_id,\n run_id=run.id,\n event_type=\"RUN_CANCELLED\",\n payload={},\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n log.reflect(\"Run cancelled\", payload={\n \"run_id\": run_id,\n })\n return run\n # #endregion cancel_run\n\n # #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status]\n # @BRIEF Get run status with statistics and event invariant checks.\n # @PRE run_id exists.\n # @POST Returns dict with run details.\n def get_run_status(self, run_id: str) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator.get_run_status\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n # Count batches\n batch_count = (\n self.db.query(TranslationBatch)\n .filter(TranslationBatch.run_id == run_id)\n .count()\n )\n\n # Get event invariants (lightweight — only fetches event_type column)\n invariants = self.event_log.get_run_event_invariants_lightweight(run_id)\n\n return {\n \"id\": run.id,\n \"job_id\": run.job_id,\n \"status\": run.status,\n \"started_at\": run.started_at.isoformat() if run.started_at else None,\n \"completed_at\": run.completed_at.isoformat() if run.completed_at else None,\n \"error_message\": run.error_message,\n \"total_records\": run.total_records or 0,\n \"successful_records\": run.successful_records or 0,\n \"failed_records\": run.failed_records or 0,\n \"skipped_records\": run.skipped_records or 0,\n \"insert_status\": run.insert_status,\n \"superset_execution_id\": run.superset_execution_id,\n \"batch_count\": batch_count,\n \"event_invariants\": invariants,\n \"created_by\": run.created_by,\n \"created_at\": run.created_at.isoformat() if run.created_at else None,\n }\n # #endregion get_run_status\n\n # #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records]\n # @BRIEF Get paginated records for a run with optional status filter.\n # @PRE run_id exists.\n # @POST Returns dict with records and pagination info.\n def get_run_records(\n self,\n run_id: str,\n page: int = 1,\n page_size: int = 50,\n status_filter: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator.get_run_records\"):\n query = self.db.query(TranslationRecord).filter(\n TranslationRecord.run_id == run_id\n )\n\n if status_filter:\n query = query.filter(TranslationRecord.status == status_filter)\n\n total = query.count()\n offset = (page - 1) * page_size\n records = (\n query.order_by(TranslationRecord.created_at.desc())\n .offset(offset)\n .limit(page_size)\n .all()\n )\n\n return {\n \"items\": [\n {\n \"id\": r.id,\n \"batch_id\": r.batch_id,\n \"source_sql\": r.source_sql,\n \"target_sql\": r.target_sql,\n \"source_object_type\": r.source_object_type,\n \"source_object_id\": r.source_object_id,\n \"source_object_name\": r.source_object_name,\n \"status\": r.status,\n \"error_message\": r.error_message,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n }\n for r in records\n ],\n \"total\": total,\n \"page\": page,\n \"page_size\": page_size,\n \"status_filter\": status_filter,\n }\n # #endregion get_run_records\n\n # #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history]\n # @BRIEF Get paginated run history for a job.\n # @PRE job_id exists.\n # @POST Returns tuple of (total_count, list_of_runs).\n def get_run_history(\n self,\n job_id: str,\n page: int = 1,\n page_size: int = 20,\n ) -> Tuple[int, List[Dict[str, Any]]]:\n with belief_scope(\"TranslationOrchestrator.get_run_history\"):\n query = self.db.query(TranslationRun).filter(\n TranslationRun.job_id == job_id\n )\n total = query.count()\n runs = (\n query.order_by(TranslationRun.created_at.desc())\n .offset((page - 1) * page_size)\n .limit(page_size)\n .all()\n )\n\n return total, [\n {\n \"id\": r.id,\n \"job_id\": r.job_id,\n \"status\": r.status,\n \"started_at\": r.started_at.isoformat() if r.started_at else None,\n \"completed_at\": r.completed_at.isoformat() if r.completed_at else None,\n \"error_message\": r.error_message,\n \"total_records\": r.total_records or 0,\n \"successful_records\": r.successful_records or 0,\n \"failed_records\": r.failed_records or 0,\n \"skipped_records\": r.skipped_records or 0,\n \"insert_status\": r.insert_status,\n \"superset_execution_id\": r.superset_execution_id,\n \"created_by\": r.created_by,\n \"created_at\": r.created_at.isoformat() if r.created_at else None,\n }\n for r in runs\n ]\n # #endregion get_run_history\n\n # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]\n # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison.\n @staticmethod\n def _compute_config_hash(job: TranslationJob) -> str:\n import hashlib\n config_str = json.dumps({\n \"source_dialect\": job.source_dialect,\n \"target_dialect\": job.target_dialect,\n \"source_datasource_id\": job.source_datasource_id,\n \"translation_column\": job.translation_column,\n \"context_columns\": job.context_columns,\n \"target_language\": job.target_language,\n \"provider_id\": job.provider_id,\n \"batch_size\": job.batch_size,\n \"upsert_strategy\": job.upsert_strategy,\n }, sort_keys=True)\n return hashlib.sha256(config_str.encode()).hexdigest()[:16]\n # #endregion _compute_config_hash\n\n # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash]\n # @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison.\n def _compute_dict_snapshot_hash(self, job_id: str) -> str:\n import hashlib\n from ...models.translate import TranslationJobDictionary\n dict_links = (\n self.db.query(TranslationJobDictionary)\n .filter(TranslationJobDictionary.job_id == job_id)\n .all()\n )\n dict_ids = sorted([dl.dictionary_id for dl in dict_links])\n hash_input = \",\".join(dict_ids)\n return hashlib.sha256(hash_input.encode()).hexdigest()[:16]\n # #endregion _compute_dict_snapshot_hash\n\n\n# #endregion TranslationOrchestrator\n# #endregion TranslationOrchestrator\n" }, { "contract_id": "start_run", "contract_type": "Function", "file_path": "backend/src/plugins/translate/orchestrator.py", - "start_line": 66, - "end_line": 166, + "start_line": 68, + "end_line": 168, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -65165,13 +66174,13 @@ } ], "anchor_syntax": "region", - "body": " # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]\n # @BRIEF Start a new translation run for a job with config snapshot and hash computation.\n # @PRE job_id exists. For manual runs, there must be an accepted preview session.\n # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.\n # @SIDE_EFFECT DB writes; event logging.\n def start_run(\n self,\n job_id: str,\n is_scheduled: bool = False,\n trigger_type: Optional[str] = None,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.start_run\"):\n # Load and validate job\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n self._job = job\n\n logger.reason(\"Starting translation run\", {\n \"job_id\": job_id,\n \"is_scheduled\": is_scheduled,\n })\n\n # Validate preconditions\n self._validate_preconditions(job, is_scheduled=is_scheduled)\n\n # Compute hashes\n config_hash = self._compute_config_hash(job)\n dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)\n\n # Build config snapshot\n config_snapshot = {\n \"source_dialect\": job.source_dialect,\n \"target_dialect\": job.target_dialect,\n \"database_dialect\": job.database_dialect,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n \"target_schema\": job.target_schema,\n \"target_table\": job.target_table,\n \"source_key_cols\": job.source_key_cols,\n \"target_key_cols\": job.target_key_cols,\n \"translation_column\": job.translation_column,\n \"context_columns\": job.context_columns,\n \"target_language\": job.target_language,\n \"provider_id\": job.provider_id,\n \"batch_size\": job.batch_size,\n \"upsert_strategy\": job.upsert_strategy,\n \"dictionary_ids\": self._compute_dict_snapshot_hash(job_id),\n }\n\n # Compute key_hash from source_key_cols\n import hashlib\n key_hash_input = json.dumps({\n \"source_key_cols\": job.source_key_cols,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n }, sort_keys=True)\n key_hash = hashlib.sha256(key_hash_input.encode()).hexdigest()[:16]\n\n # Create run record with all hash/snapshot fields\n run = TranslationRun(\n id=str(uuid.uuid4()),\n job_id=job_id,\n status=\"PENDING\",\n trigger_type=trigger_type or (\"scheduled\" if is_scheduled else \"manual\"),\n config_snapshot=config_snapshot,\n key_hash=key_hash,\n config_hash=config_hash,\n dict_snapshot_hash=dict_snapshot_hash,\n created_by=self.current_user,\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(run)\n self.db.flush()\n\n # Record event\n self.event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\n \"is_scheduled\": is_scheduled,\n \"trigger_type\": run.trigger_type,\n \"config_hash\": config_hash,\n \"dict_snapshot_hash\": dict_snapshot_hash,\n \"key_hash\": key_hash,\n },\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n logger.reflect(\"Run created\", {\n \"run_id\": run.id,\n \"job_id\": job_id,\n \"status\": run.status,\n \"trigger_type\": run.trigger_type,\n })\n return run\n # #endregion start_run\n" + "body": " # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create]\n # @BRIEF Start a new translation run for a job with config snapshot and hash computation.\n # @PRE job_id exists. For manual runs, there must be an accepted preview session.\n # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded.\n # @SIDE_EFFECT DB writes; event logging.\n def start_run(\n self,\n job_id: str,\n is_scheduled: bool = False,\n trigger_type: Optional[str] = None,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.start_run\"):\n # Load and validate job\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n self._job = job\n\n log.reason(\"Starting translation run\", payload={\n \"job_id\": job_id,\n \"is_scheduled\": is_scheduled,\n })\n\n # Validate preconditions\n self._validate_preconditions(job, is_scheduled=is_scheduled)\n\n # Compute hashes\n config_hash = self._compute_config_hash(job)\n dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)\n\n # Build config snapshot\n config_snapshot = {\n \"source_dialect\": job.source_dialect,\n \"target_dialect\": job.target_dialect,\n \"database_dialect\": job.database_dialect,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n \"target_schema\": job.target_schema,\n \"target_table\": job.target_table,\n \"source_key_cols\": job.source_key_cols,\n \"target_key_cols\": job.target_key_cols,\n \"translation_column\": job.translation_column,\n \"context_columns\": job.context_columns,\n \"target_language\": job.target_language,\n \"provider_id\": job.provider_id,\n \"batch_size\": job.batch_size,\n \"upsert_strategy\": job.upsert_strategy,\n \"dictionary_ids\": self._compute_dict_snapshot_hash(job_id),\n }\n\n # Compute key_hash from source_key_cols\n import hashlib\n key_hash_input = json.dumps({\n \"source_key_cols\": job.source_key_cols,\n \"source_datasource_id\": job.source_datasource_id,\n \"source_table\": job.source_table,\n }, sort_keys=True)\n key_hash = hashlib.sha256(key_hash_input.encode()).hexdigest()[:16]\n\n # Create run record with all hash/snapshot fields\n run = TranslationRun(\n id=str(uuid.uuid4()),\n job_id=job_id,\n status=\"PENDING\",\n trigger_type=trigger_type or (\"scheduled\" if is_scheduled else \"manual\"),\n config_snapshot=config_snapshot,\n key_hash=key_hash,\n config_hash=config_hash,\n dict_snapshot_hash=dict_snapshot_hash,\n created_by=self.current_user,\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(run)\n self.db.flush()\n\n # Record event\n self.event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\n \"is_scheduled\": is_scheduled,\n \"trigger_type\": run.trigger_type,\n \"config_hash\": config_hash,\n \"dict_snapshot_hash\": dict_snapshot_hash,\n \"key_hash\": key_hash,\n },\n created_by=self.current_user,\n )\n\n self.db.commit()\n self.db.refresh(run)\n\n log.reflect(\"Run created\", payload={\n \"run_id\": run.id,\n \"job_id\": job_id,\n \"status\": run.status,\n \"trigger_type\": run.trigger_type,\n })\n return run\n # #endregion start_run\n" }, { "contract_id": "_generate_and_insert_sql", "contract_type": "Function", "file_path": "backend/src/plugins/translate/orchestrator.py", - "start_line": 302, + "start_line": 303, "end_line": 433, "tier": "TIER_3", "complexity": 5, @@ -65228,7 +66237,7 @@ } ], "anchor_syntax": "region", - "body": " # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]\n # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.\n # @PRE job has target table configured. run has successful records.\n # @POST SQL is generated and submitted. Returns execution result.\n # @SIDE_EFFECT Superset API call; event logging.\n def _generate_and_insert_sql(\n self,\n job: TranslationJob,\n run: TranslationRun,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator._generate_and_insert_sql\"):\n # Fetch successful records\n records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run.id,\n TranslationRecord.status == \"SUCCESS\",\n TranslationRecord.target_sql.isnot(None),\n )\n .all()\n )\n\n if not records:\n logger.reason(\"No successful records to insert\", {\"run_id\": run.id})\n return {\"status\": \"skipped\", \"reason\": \"no_records\", \"query_id\": None}\n\n logger.reason(f\"Generating SQL for {len(records)} records\", {\n \"run_id\": run.id,\n \"dialect\": job.database_dialect or job.target_dialect,\n })\n\n # Build rows for SQL generation\n # Only include the translation column — context columns are for LLM only\n columns = []\n if job.translation_column:\n columns.append(job.translation_column)\n\n # Include key columns if present (for UPSERT matching)\n if job.target_key_cols:\n for k in job.target_key_cols:\n if k not in columns:\n columns.append(k)\n\n rows_for_sql = []\n for rec in records:\n row_data = {}\n if job.translation_column:\n row_data[job.translation_column] = rec.target_sql or \"\"\n if job.target_key_cols:\n source_data = rec.source_data or {}\n for k in job.target_key_cols:\n # Use source_data value if available, fall back to empty string\n row_data[k] = source_data.get(k, \"\")\n rows_for_sql.append(row_data)\n\n if not columns:\n # Use target_sql as the sole column\n columns = [job.translation_column or \"translated_text\"]\n rows_for_sql = [{columns[0]: rec.target_sql or \"\"} for rec in records]\n\n # Generate SQL\n try:\n sql, row_count = SQLGenerator.generate(\n dialect=job.database_dialect or job.target_dialect or \"postgresql\",\n target_schema=job.target_schema,\n target_table=job.target_table or \"translated_data\",\n columns=columns,\n rows=rows_for_sql,\n key_columns=job.target_key_cols,\n upsert_strategy=job.upsert_strategy or \"MERGE\",\n )\n except ValueError as e:\n logger.explore(\"SQL generation failed\", {\"error\": str(e)})\n return {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase start\n logger.reason(\"Insert SQL generated\", {\n \"run_id\": run.id,\n \"sql_preview\": sql[:500],\n \"sql_length\": len(sql),\n \"row_count\": row_count,\n \"dialect\": job.database_dialect or job.target_dialect or \"postgresql\",\n \"columns\": columns,\n \"has_key_cols\": bool(job.target_key_cols),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_STARTED\",\n payload={\"sql_length\": len(sql), \"row_count\": row_count},\n created_by=self.current_user,\n )\n\n # Submit to Superset\n try:\n env_id = job.environment_id or job.source_dialect or \"\"\n # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment\n target_db_id = None\n if job.target_database_id:\n try:\n target_db_id = int(job.target_database_id)\n except (ValueError, TypeError):\n # Could be a UUID — pass as-is, executor will resolve it\n target_db_id = job.target_database_id\n logger.reason(\"target_database_id is not an integer, will resolve as UUID\", {\n \"target_database_id\": job.target_database_id,\n })\n executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)\n result = executor.execute_and_poll(\n sql=sql,\n max_polls=30,\n poll_interval_seconds=2.0,\n )\n except Exception as e:\n logger.explore(\"Superset SQL submission failed\", {\n \"run_id\": run.id,\n \"error\": str(e),\n })\n result = {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase complete\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_COMPLETED\",\n payload=result,\n created_by=self.current_user,\n )\n\n return result\n # #endregion _generate_and_insert_sql\n" + "body": " # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert]\n # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab.\n # @PRE job has target table configured. run has successful records.\n # @POST SQL is generated and submitted. Returns execution result.\n # @SIDE_EFFECT Superset API call; event logging.\n def _generate_and_insert_sql(\n self,\n job: TranslationJob,\n run: TranslationRun,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationOrchestrator._generate_and_insert_sql\"):\n # Fetch successful records\n records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.run_id == run.id,\n TranslationRecord.status == \"SUCCESS\",\n TranslationRecord.target_sql.isnot(None),\n )\n .all()\n )\n\n if not records:\n log.reason(\"No successful records to insert\", payload={\"run_id\": run.id})\n return {\"status\": \"skipped\", \"reason\": \"no_records\", \"query_id\": None}\n\n log.reason(f\"Generating SQL for {len(records)} records\", payload={\n \"run_id\": run.id,\n \"dialect\": job.database_dialect or job.target_dialect,\n })\n\n # Build rows for SQL generation\n # Only include the translation column — context columns are for LLM only\n columns = []\n if job.translation_column:\n columns.append(job.translation_column)\n\n # Include key columns if present (for UPSERT matching)\n if job.target_key_cols:\n for k in job.target_key_cols:\n if k not in columns:\n columns.append(k)\n\n rows_for_sql = []\n for rec in records:\n row_data = {}\n if job.translation_column:\n row_data[job.translation_column] = rec.target_sql or \"\"\n if job.target_key_cols:\n source_data = rec.source_data or {}\n for k in job.target_key_cols:\n # Use source_data value if available, fall back to empty string\n row_data[k] = source_data.get(k, \"\")\n rows_for_sql.append(row_data)\n\n if not columns:\n # Use target_sql as the sole column\n columns = [job.translation_column or \"translated_text\"]\n rows_for_sql = [{columns[0]: rec.target_sql or \"\"} for rec in records]\n\n # Generate SQL\n try:\n sql, row_count = SQLGenerator.generate(\n dialect=job.database_dialect or job.target_dialect or \"postgresql\",\n target_schema=job.target_schema,\n target_table=job.target_table or \"translated_data\",\n columns=columns,\n rows=rows_for_sql,\n key_columns=job.target_key_cols,\n upsert_strategy=job.upsert_strategy or \"MERGE\",\n )\n except ValueError as e:\n log.explore(\"SQL generation failed\", error=str(e))\n return {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase start\n log.reason(\"Insert SQL generated\", payload={\n \"run_id\": run.id,\n \"sql_preview\": sql[:500],\n \"sql_length\": len(sql),\n \"row_count\": row_count,\n \"dialect\": job.database_dialect or job.target_dialect or \"postgresql\",\n \"columns\": columns,\n \"has_key_cols\": bool(job.target_key_cols),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_STARTED\",\n payload={\"sql_length\": len(sql), \"row_count\": row_count},\n created_by=self.current_user,\n )\n\n # Submit to Superset\n try:\n env_id = job.environment_id or job.source_dialect or \"\"\n # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment\n target_db_id = None\n if job.target_database_id:\n try:\n target_db_id = int(job.target_database_id)\n except (ValueError, TypeError):\n # Could be a UUID — pass as-is, executor will resolve it\n target_db_id = job.target_database_id\n log.reason(\"target_database_id is not an integer, will resolve as UUID\", payload={\n \"target_database_id\": job.target_database_id,\n })\n executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id)\n result = executor.execute_and_poll(\n sql=sql,\n max_polls=30,\n poll_interval_seconds=2.0,\n )\n except Exception as e:\n log.explore(\"Superset SQL submission failed\", error=str(e), payload={\n \"run_id\": run.id,\n })\n result = {\"status\": \"failed\", \"error_message\": str(e), \"query_id\": None}\n\n # Log insert phase complete\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"INSERT_PHASE_COMPLETED\",\n payload=result,\n created_by=self.current_user,\n )\n\n return result\n # #endregion _generate_and_insert_sql\n" }, { "contract_id": "_validate_preconditions", @@ -65291,7 +66300,7 @@ } ], "anchor_syntax": "region", - "body": " # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]\n # @BRIEF Validate preconditions before starting a translation run.\n # @PRE None.\n # @POST Raises ValueError if preconditions are not met.\n # @SIDE_EFFECT None.\n def _validate_preconditions(\n self,\n job: TranslationJob,\n is_scheduled: bool = False,\n ) -> None:\n with belief_scope(\"TranslationOrchestrator._validate_preconditions\"):\n # Job must be in valid status\n if job.status in (\"DRAFT\",):\n raise ValueError(\n f\"Cannot run job '{job.id}' in status '{job.status}'. \"\n f\"Job must be READY, ACTIVE, or COMPLETED.\"\n )\n\n # Must have target table configured\n if not job.target_table:\n raise ValueError(\n f\"Job '{job.id}' has no target table configured. \"\n \"Configure a target table before running.\"\n )\n\n # Must have LLM provider configured\n if not job.provider_id:\n raise ValueError(\n f\"Job '{job.id}' has no LLM provider configured. \"\n \"Select an LLM provider before running.\"\n )\n\n # Must have a translation column\n if not job.translation_column:\n raise ValueError(\n f\"Job '{job.id}' has no translation column configured. \"\n \"Select a translation column before running.\"\n )\n\n # For manual runs, must have accepted preview\n if not is_scheduled:\n accepted_session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job.id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not accepted_session:\n raise ValueError(\n f\"Job '{job.id}' has no accepted preview session. \"\n \"Run and accept a preview before executing a manual translation run.\"\n )\n\n logger.reason(\"Preconditions validated\", {\n \"job_id\": job.id,\n \"is_scheduled\": is_scheduled,\n })\n # #endregion _validate_preconditions\n" + "body": " # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation]\n # @BRIEF Validate preconditions before starting a translation run.\n # @PRE None.\n # @POST Raises ValueError if preconditions are not met.\n # @SIDE_EFFECT None.\n def _validate_preconditions(\n self,\n job: TranslationJob,\n is_scheduled: bool = False,\n ) -> None:\n with belief_scope(\"TranslationOrchestrator._validate_preconditions\"):\n # Job must be in valid status\n if job.status in (\"DRAFT\",):\n raise ValueError(\n f\"Cannot run job '{job.id}' in status '{job.status}'. \"\n f\"Job must be READY, ACTIVE, or COMPLETED.\"\n )\n\n # Must have target table configured\n if not job.target_table:\n raise ValueError(\n f\"Job '{job.id}' has no target table configured. \"\n \"Configure a target table before running.\"\n )\n\n # Must have LLM provider configured\n if not job.provider_id:\n raise ValueError(\n f\"Job '{job.id}' has no LLM provider configured. \"\n \"Select an LLM provider before running.\"\n )\n\n # Must have a translation column\n if not job.translation_column:\n raise ValueError(\n f\"Job '{job.id}' has no translation column configured. \"\n \"Select a translation column before running.\"\n )\n\n # For manual runs, must have accepted preview\n if not is_scheduled:\n accepted_session = (\n self.db.query(TranslationPreviewSession)\n .filter(\n TranslationPreviewSession.job_id == job.id,\n TranslationPreviewSession.status == \"APPLIED\",\n )\n .order_by(TranslationPreviewSession.created_at.desc())\n .first()\n )\n if not accepted_session:\n raise ValueError(\n f\"Job '{job.id}' has no accepted preview session. \"\n \"Run and accept a preview before executing a manual translation run.\"\n )\n\n log.reason(\"Preconditions validated\", payload={\n \"job_id\": job.id,\n \"is_scheduled\": is_scheduled,\n })\n # #endregion _validate_preconditions\n" }, { "contract_id": "retry_failed_batches", @@ -65354,7 +66363,7 @@ } ], "anchor_syntax": "region", - "body": " # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]\n # @BRIEF Retry failed batches in a run by re-processing failed records.\n # @PRE run exists and has failed batches.\n # @POST Failed batches are re-processed; run status and stats updated.\n # @SIDE_EFFECT LLM calls; DB writes; event logging.\n def retry_failed_batches(\n self,\n run_id: str,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.retry_failed_batches\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n # Find failed batches\n failed_batches = (\n self.db.query(TranslationBatch)\n .filter(\n TranslationBatch.run_id == run_id,\n TranslationBatch.status.in_([\"FAILED\", \"COMPLETED_WITH_ERRORS\"]),\n )\n .all()\n )\n\n if not failed_batches:\n raise ValueError(f\"No failed batches found for run '{run_id}'\")\n\n logger.reason(\"Retrying failed batches\", {\n \"run_id\": run_id,\n \"batch_count\": len(failed_batches),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_RETRYING\",\n payload={\"batch_count\": len(failed_batches)},\n created_by=self.current_user,\n )\n\n # Re-process each failed batch\n executor = TranslationExecutor(self.db, self.config_manager, self.current_user)\n run.status = \"RUNNING\"\n self.db.flush()\n\n for batch in failed_batches:\n # Fetch failed records for this batch\n failed_records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.batch_id == batch.id,\n TranslationRecord.status == \"FAILED\",\n )\n .all()\n )\n\n if not failed_records:\n continue\n\n # Build rows from failed records\n retry_rows = []\n for rec in failed_records:\n retry_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": None,\n \"source_object_name\": rec.source_object_name or \"\",\n })\n\n # Process retry batch\n result = executor._process_batch(\n job=job,\n run_id=run_id,\n batch_index=batch.batch_index,\n batch_rows=retry_rows,\n )\n\n # Update run stats\n run.successful_records = (run.successful_records or 0) + result[\"successful\"]\n run.failed_records = (run.failed_records or 0) + result[\"failed\"]\n run.skipped_records = (run.skipped_records or 0) + result[\"skipped\"]\n self.db.flush()\n\n # Update run status\n if run.failed_records == 0:\n run.status = \"COMPLETED\"\n elif run.successful_records == 0:\n run.status = \"FAILED\"\n else:\n run.status = \"COMPLETED\"\n\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_COMPLETED\",\n payload={\"retry\": True},\n created_by=self.current_user,\n )\n\n self.db.commit()\n logger.reflect(\"Retry complete\", {\n \"run_id\": run_id,\n \"status\": run.status,\n })\n return run\n # #endregion retry_failed_batches\n" + "body": " # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch]\n # @BRIEF Retry failed batches in a run by re-processing failed records.\n # @PRE run exists and has failed batches.\n # @POST Failed batches are re-processed; run status and stats updated.\n # @SIDE_EFFECT LLM calls; DB writes; event logging.\n def retry_failed_batches(\n self,\n run_id: str,\n ) -> TranslationRun:\n with belief_scope(\"TranslationOrchestrator.retry_failed_batches\"):\n run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first()\n if not run:\n raise ValueError(f\"Run '{run_id}' not found\")\n\n job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()\n if not job:\n raise ValueError(f\"Job '{run.job_id}' not found\")\n self._job = job\n\n # Find failed batches\n failed_batches = (\n self.db.query(TranslationBatch)\n .filter(\n TranslationBatch.run_id == run_id,\n TranslationBatch.status.in_([\"FAILED\", \"COMPLETED_WITH_ERRORS\"]),\n )\n .all()\n )\n\n if not failed_batches:\n raise ValueError(f\"No failed batches found for run '{run_id}'\")\n\n log.reason(\"Retrying failed batches\", payload={\n \"run_id\": run_id,\n \"batch_count\": len(failed_batches),\n })\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_RETRYING\",\n payload={\"batch_count\": len(failed_batches)},\n created_by=self.current_user,\n )\n\n # Re-process each failed batch\n executor = TranslationExecutor(self.db, self.config_manager, self.current_user)\n run.status = \"RUNNING\"\n self.db.flush()\n\n for batch in failed_batches:\n # Fetch failed records for this batch\n failed_records = (\n self.db.query(TranslationRecord)\n .filter(\n TranslationRecord.batch_id == batch.id,\n TranslationRecord.status == \"FAILED\",\n )\n .all()\n )\n\n if not failed_records:\n continue\n\n # Build rows from failed records\n retry_rows = []\n for rec in failed_records:\n retry_rows.append({\n \"row_index\": rec.source_object_id or \"0\",\n \"source_text\": rec.source_sql or \"\",\n \"approved_translation\": None,\n \"source_object_name\": rec.source_object_name or \"\",\n })\n\n # Process retry batch\n result = executor._process_batch(\n job=job,\n run_id=run_id,\n batch_index=batch.batch_index,\n batch_rows=retry_rows,\n )\n\n # Update run stats\n run.successful_records = (run.successful_records or 0) + result[\"successful\"]\n run.failed_records = (run.failed_records or 0) + result[\"failed\"]\n run.skipped_records = (run.skipped_records or 0) + result[\"skipped\"]\n self.db.flush()\n\n # Update run status\n if run.failed_records == 0:\n run.status = \"COMPLETED\"\n elif run.successful_records == 0:\n run.status = \"FAILED\"\n else:\n run.status = \"COMPLETED\"\n\n run.completed_at = datetime.now(timezone.utc)\n self.db.flush()\n\n self.event_log.log_event(\n job_id=job.id,\n run_id=run.id,\n event_type=\"RUN_COMPLETED\",\n payload={\"retry\": True},\n created_by=self.current_user,\n )\n\n self.db.commit()\n log.reflect(\"Retry complete\", payload={\n \"run_id\": run_id,\n \"status\": run.status,\n })\n return run\n # #endregion retry_failed_batches\n" }, { "contract_id": "_compute_config_hash", @@ -65467,8 +66476,8 @@ "contract_id": "DEFAULT_EXECUTION_PROMPT_TEMPLATE", "contract_type": "Constant", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 41, - "end_line": 54, + "start_line": 44, + "end_line": 57, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -65508,8 +66517,8 @@ "contract_id": "DEFAULT_PREVIEW_PROMPT_TEMPLATE", "contract_type": "Constant", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 57, - "end_line": 71, + "start_line": 60, + "end_line": 74, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -65549,8 +66558,8 @@ "contract_id": "TokenEstimator", "contract_type": "Class", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 74, - "end_line": 105, + "start_line": 77, + "end_line": 108, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -65582,8 +66591,8 @@ "contract_id": "estimate_prompt_tokens", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 83, - "end_line": 89, + "start_line": 86, + "end_line": 92, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -65608,8 +66617,8 @@ "contract_id": "estimate_output_tokens", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 91, - "end_line": 95, + "start_line": 94, + "end_line": 98, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -65634,8 +66643,8 @@ "contract_id": "estimate_cost", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 97, - "end_line": 102, + "start_line": 100, + "end_line": 105, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -65660,8 +66669,8 @@ "contract_id": "preview_rows", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 127, - "end_line": 339, + "start_line": 130, + "end_line": 342, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -65699,14 +66708,14 @@ } ], "anchor_syntax": "region", - "body": " # #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows]\n # @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.\n # @PRE job_id exists and job has source_datasource_id, translation_column configured.\n # @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session.\n # @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.\n def preview_rows(\n self,\n job_id: str,\n sample_size: int = 10,\n prompt_template: Optional[str] = None,\n env_id: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationPreview.preview_rows\"):\n logger.reason(\"Starting preview for job\", {\"job_id\": job_id, \"sample_size\": sample_size})\n\n # 1. Load job\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n if not job.source_datasource_id:\n raise ValueError(\"Job must have a source datasource configured for preview\")\n if not job.translation_column:\n raise ValueError(\"Job must have a translation column configured for preview\")\n\n # 2. Compute config hash and dict snapshot hash\n config_hash = self._compute_config_hash(job)\n dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)\n\n # 3. Fetch sample rows from Superset\n logger.reason(\"Fetching sample rows from Superset\", {\n \"datasource_id\": job.source_datasource_id,\n \"sample_size\": sample_size,\n \"translation_column\": job.translation_column,\n })\n source_rows = self._fetch_sample_rows(\n job=job,\n sample_size=sample_size,\n env_id=env_id,\n )\n if not source_rows:\n raise ValueError(\"No rows returned from datasource for preview\")\n\n actual_row_count = len(source_rows)\n logger.reason(f\"Fetched {actual_row_count} sample row(s)\")\n\n # Debug: log first row keys and translation column value\n if source_rows:\n first_row = source_rows[0]\n logger.reason(\n f\"First source row keys={list(first_row.keys())} \"\n f\"translation_col={job.translation_column} \"\n f\"val='{first_row.get(job.translation_column, '')}'\"\n )\n\n # 4. Build prompt context from rows\n all_source_texts = []\n row_meta: List[Dict[str, Any]] = []\n for idx, row in enumerate(source_rows):\n translation_value = str(row.get(job.translation_column, \"\") or \"\")\n context_values = {}\n if job.context_columns:\n for col in job.context_columns:\n context_values[col] = str(row.get(col, \"\") or \"\")\n all_source_texts.append(translation_value)\n row_meta.append({\n \"row_index\": idx,\n \"source_text\": translation_value,\n \"context_data\": context_values,\n \"source_row\": row,\n })\n\n # 5. Filter dictionary entries for this batch\n dict_matches = DictionaryManager.filter_for_batch(\n self.db, all_source_texts, job_id\n )\n\n # Build dictionary glossary section\n dictionary_section = \"\"\n if dict_matches:\n glossary_lines = []\n for m in dict_matches:\n glossary_lines.append(\n f\"- '{m['source_term']}' -> '{m['target_term']}'\"\n f\"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}\"\n )\n dictionary_section = (\n \"Terminology dictionary (use these translations when applicable):\\n\"\n + \"\\n\".join(glossary_lines)\n + \"\\n\\n\"\n )\n\n # 6. Build LLM prompt\n rows_json = json.dumps([\n {\"row_id\": str(m[\"row_index\"]), \"text\": m[\"source_text\"], \"context\": m[\"context_data\"]}\n for m in row_meta\n ], indent=2)\n\n template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE\n prompt = render_prompt(template, {\n \"source_language\": job.source_dialect or \"SQL\",\n \"target_language\": job.target_language or job.target_dialect or \"en\",\n \"source_dialect\": job.source_dialect or \"\",\n \"target_dialect\": job.target_dialect or \"\",\n \"translation_column\": job.translation_column or \"\",\n \"dictionary_section\": dictionary_section,\n \"rows_json\": rows_json,\n \"row_count\": str(actual_row_count),\n })\n\n # 7. Estimate tokens/cost for sample and full dataset\n sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)\n sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)\n sample_total_tokens = sample_prompt_tokens + sample_output_tokens\n sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)\n\n # Estimate full dataset cost (if we knew total rows)\n total_est_tokens = TokenEstimator.estimate_prompt_tokens(\n prompt.replace(str(actual_row_count), \"{total}\")\n ) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation\n total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)\n\n # 8. Call LLM\n logger.reason(\"Calling LLM for preview translation\", {\n \"provider_id\": job.provider_id,\n \"row_count\": actual_row_count,\n \"estimated_tokens\": sample_total_tokens,\n })\n llm_response = self._call_llm(\n job=job,\n prompt=prompt,\n )\n\n # 9. Parse LLM response\n translations = self._parse_llm_response(llm_response, actual_row_count)\n\n # 10. Create preview session\n session = TranslationPreviewSession(\n id=str(uuid.uuid4()),\n job_id=job_id,\n status=\"ACTIVE\",\n created_by=self.current_user,\n created_at=datetime.now(timezone.utc),\n expires_at=datetime.now(timezone.utc) + timedelta(hours=24),\n )\n self.db.add(session)\n self.db.flush()\n\n # 11. Create preview records\n records = []\n for meta in row_meta:\n idx = meta[\"row_index\"]\n translation = translations.get(str(idx), \"\")\n is_rejected = False\n status = \"PENDING\"\n feedback = None\n\n record = TranslationPreviewRecord(\n id=str(uuid.uuid4()),\n session_id=session.id,\n source_sql=meta[\"source_text\"],\n target_sql=translation,\n source_object_type=\"table_row\",\n source_object_id=str(idx),\n source_object_name=f\"Row {idx + 1}\",\n status=status,\n feedback=feedback,\n source_data=meta.get(\"context_data\"),\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(record)\n self.db.flush()\n records.append({\n \"id\": record.id,\n \"source_sql\": record.source_sql,\n \"target_sql\": record.target_sql,\n \"source_object_type\": record.source_object_type,\n \"source_object_id\": record.source_object_id,\n \"source_object_name\": record.source_object_name,\n \"status\": record.status,\n \"feedback\": record.feedback,\n })\n\n self.db.commit()\n\n result = {\n \"id\": session.id,\n \"job_id\": job_id,\n \"status\": \"ACTIVE\",\n \"created_by\": self.current_user,\n \"created_at\": session.created_at.isoformat(),\n \"expires_at\": session.expires_at.isoformat() if session.expires_at else None,\n \"records\": records,\n \"cost_estimate\": {\n \"sample_size\": actual_row_count,\n \"sample_prompt_tokens\": sample_prompt_tokens,\n \"sample_output_tokens\": sample_output_tokens,\n \"sample_total_tokens\": sample_total_tokens,\n \"sample_cost\": sample_cost,\n \"estimated_total_rows\": actual_row_count * 10,\n \"estimated_tokens\": total_est_tokens,\n \"estimated_cost\": total_est_cost,\n },\n \"config_hash\": config_hash,\n \"dict_snapshot_hash\": dict_snapshot_hash,\n }\n\n logger.reflect(\"Preview completed\", {\n \"session_id\": session.id,\n \"row_count\": actual_row_count,\n \"sample_cost\": sample_cost,\n })\n return result\n # #endregion preview_rows\n" + "body": " # #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows]\n # @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.\n # @PRE job_id exists and job has source_datasource_id, translation_column configured.\n # @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session.\n # @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.\n def preview_rows(\n self,\n job_id: str,\n sample_size: int = 10,\n prompt_template: Optional[str] = None,\n env_id: Optional[str] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"TranslationPreview.preview_rows\"):\n log.reason(\"Starting preview for job\", payload={\"job_id\": job_id, \"sample_size\": sample_size})\n\n # 1. Load job\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n if not job.source_datasource_id:\n raise ValueError(\"Job must have a source datasource configured for preview\")\n if not job.translation_column:\n raise ValueError(\"Job must have a translation column configured for preview\")\n\n # 2. Compute config hash and dict snapshot hash\n config_hash = self._compute_config_hash(job)\n dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)\n\n # 3. Fetch sample rows from Superset\n log.reason(\"Fetching sample rows from Superset\", payload={\n \"datasource_id\": job.source_datasource_id,\n \"sample_size\": sample_size,\n \"translation_column\": job.translation_column,\n })\n source_rows = self._fetch_sample_rows(\n job=job,\n sample_size=sample_size,\n env_id=env_id,\n )\n if not source_rows:\n raise ValueError(\"No rows returned from datasource for preview\")\n\n actual_row_count = len(source_rows)\n log.reason(f\"Fetched {actual_row_count} sample row(s)\")\n\n # Debug: log first row keys and translation column value\n if source_rows:\n first_row = source_rows[0]\n log.reason(\n f\"First source row keys={list(first_row.keys())} \"\n f\"translation_col={job.translation_column} \"\n f\"val='{first_row.get(job.translation_column, '')}'\"\n )\n\n # 4. Build prompt context from rows\n all_source_texts = []\n row_meta: List[Dict[str, Any]] = []\n for idx, row in enumerate(source_rows):\n translation_value = str(row.get(job.translation_column, \"\") or \"\")\n context_values = {}\n if job.context_columns:\n for col in job.context_columns:\n context_values[col] = str(row.get(col, \"\") or \"\")\n all_source_texts.append(translation_value)\n row_meta.append({\n \"row_index\": idx,\n \"source_text\": translation_value,\n \"context_data\": context_values,\n \"source_row\": row,\n })\n\n # 5. Filter dictionary entries for this batch\n dict_matches = DictionaryManager.filter_for_batch(\n self.db, all_source_texts, job_id\n )\n\n # Build dictionary glossary section\n dictionary_section = \"\"\n if dict_matches:\n glossary_lines = []\n for m in dict_matches:\n glossary_lines.append(\n f\"- '{m['source_term']}' -> '{m['target_term']}'\"\n f\"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}\"\n )\n dictionary_section = (\n \"Terminology dictionary (use these translations when applicable):\\n\"\n + \"\\n\".join(glossary_lines)\n + \"\\n\\n\"\n )\n\n # 6. Build LLM prompt\n rows_json = json.dumps([\n {\"row_id\": str(m[\"row_index\"]), \"text\": m[\"source_text\"], \"context\": m[\"context_data\"]}\n for m in row_meta\n ], indent=2)\n\n template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE\n prompt = render_prompt(template, {\n \"source_language\": job.source_dialect or \"SQL\",\n \"target_language\": job.target_language or job.target_dialect or \"en\",\n \"source_dialect\": job.source_dialect or \"\",\n \"target_dialect\": job.target_dialect or \"\",\n \"translation_column\": job.translation_column or \"\",\n \"dictionary_section\": dictionary_section,\n \"rows_json\": rows_json,\n \"row_count\": str(actual_row_count),\n })\n\n # 7. Estimate tokens/cost for sample and full dataset\n sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)\n sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)\n sample_total_tokens = sample_prompt_tokens + sample_output_tokens\n sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)\n\n # Estimate full dataset cost (if we knew total rows)\n total_est_tokens = TokenEstimator.estimate_prompt_tokens(\n prompt.replace(str(actual_row_count), \"{total}\")\n ) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation\n total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)\n\n # 8. Call LLM\n log.reason(\"Calling LLM for preview translation\", payload={\n \"provider_id\": job.provider_id,\n \"row_count\": actual_row_count,\n \"estimated_tokens\": sample_total_tokens,\n })\n llm_response = self._call_llm(\n job=job,\n prompt=prompt,\n )\n\n # 9. Parse LLM response\n translations = self._parse_llm_response(llm_response, actual_row_count)\n\n # 10. Create preview session\n session = TranslationPreviewSession(\n id=str(uuid.uuid4()),\n job_id=job_id,\n status=\"ACTIVE\",\n created_by=self.current_user,\n created_at=datetime.now(timezone.utc),\n expires_at=datetime.now(timezone.utc) + timedelta(hours=24),\n )\n self.db.add(session)\n self.db.flush()\n\n # 11. Create preview records\n records = []\n for meta in row_meta:\n idx = meta[\"row_index\"]\n translation = translations.get(str(idx), \"\")\n is_rejected = False\n status = \"PENDING\"\n feedback = None\n\n record = TranslationPreviewRecord(\n id=str(uuid.uuid4()),\n session_id=session.id,\n source_sql=meta[\"source_text\"],\n target_sql=translation,\n source_object_type=\"table_row\",\n source_object_id=str(idx),\n source_object_name=f\"Row {idx + 1}\",\n status=status,\n feedback=feedback,\n source_data=meta.get(\"context_data\"),\n created_at=datetime.now(timezone.utc),\n )\n self.db.add(record)\n self.db.flush()\n records.append({\n \"id\": record.id,\n \"source_sql\": record.source_sql,\n \"target_sql\": record.target_sql,\n \"source_object_type\": record.source_object_type,\n \"source_object_id\": record.source_object_id,\n \"source_object_name\": record.source_object_name,\n \"status\": record.status,\n \"feedback\": record.feedback,\n })\n\n self.db.commit()\n\n result = {\n \"id\": session.id,\n \"job_id\": job_id,\n \"status\": \"ACTIVE\",\n \"created_by\": self.current_user,\n \"created_at\": session.created_at.isoformat(),\n \"expires_at\": session.expires_at.isoformat() if session.expires_at else None,\n \"records\": records,\n \"cost_estimate\": {\n \"sample_size\": actual_row_count,\n \"sample_prompt_tokens\": sample_prompt_tokens,\n \"sample_output_tokens\": sample_output_tokens,\n \"sample_total_tokens\": sample_total_tokens,\n \"sample_cost\": sample_cost,\n \"estimated_total_rows\": actual_row_count * 10,\n \"estimated_tokens\": total_est_tokens,\n \"estimated_cost\": total_est_cost,\n },\n \"config_hash\": config_hash,\n \"dict_snapshot_hash\": dict_snapshot_hash,\n }\n\n log.reflect(\"Preview completed\", payload={\n \"session_id\": session.id,\n \"row_count\": actual_row_count,\n \"sample_cost\": sample_cost,\n })\n return result\n # #endregion preview_rows\n" }, { "contract_id": "get_preview_session", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 466, - "end_line": 507, + "start_line": 469, + "end_line": 510, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -65745,8 +66754,8 @@ "contract_id": "_fetch_sample_rows", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 509, - "end_line": 592, + "start_line": 512, + "end_line": 597, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -65784,14 +66793,14 @@ } ], "anchor_syntax": "region", - "body": " # #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample]\n # @BRIEF Fetch sample rows from the Superset dataset for preview.\n # @PRE job has source_datasource_id and translation_column.\n # @POST Returns list of dicts with row data from Superset chart data API.\n # @SIDE_EFFECT Calls Superset chart data endpoint with pagination.\n def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationPreview._fetch_sample_rows\"):\n # Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)\n environments = self.config_manager.get_environments()\n target_env_id = env_id or job.environment_id or job.source_dialect or \"\"\n env_config = next(\n (e for e in environments if e.id == target_env_id),\n None,\n )\n if not env_config:\n logger.explore(\"Could not find environment for datasource\", {\n \"env_id\": target_env_id,\n })\n # Fallback: try first environment\n if environments:\n env_config = environments[0]\n logger.explore(\"Falling back to first available environment\", {\n \"env_id\": env_config.id,\n })\n else:\n raise ValueError(\"No Superset environments configured\")\n\n client = SupersetClient(env_config)\n\n # Fetch dataset detail to build proper query context\n dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))\n\n # Build query context for chart data endpoint.\n # Virtual columns (e.g. comment_text_ru) are NOT resolved when:\n # - result_type=\"query\" (physical columns only)\n # - query_mode=\"raw\" (virtual columns unavailable in raw mode)\n # Solution: remove both result_type=\"query\" AND query_mode=\"raw\",\n # use aggregate mode with no metrics — this resolves virtual columns.\n query_context = client.build_dataset_preview_query_context(\n dataset_id=int(job.source_datasource_id),\n dataset_record=dataset_detail,\n template_params={},\n effective_filters=[],\n )\n\n # Modify: use result_type=\"samples\" which returns sample data\n # including all columns (physical + virtual), without needing\n # explicit column objects that trigger validation errors.\n queries = query_context.get(\"queries\", [])\n if queries:\n queries[0][\"row_limit\"] = sample_size\n queries[0].pop(\"result_type\", None)\n queries[0].pop(\"columns\", None)\n queries[0][\"metrics\"] = []\n query_context[\"result_type\"] = \"samples\"\n form_data = query_context.get(\"form_data\", {})\n form_data.pop(\"query_mode\", None)\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/chart/data\",\n data=json.dumps(query_context),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n logger.explore(\"Chart data API failed\", {\"error\": str(e)})\n raise ValueError(f\"Failed to fetch sample data from Superset: {e}\")\n\n # Parse response\n rows = self._extract_data_rows(response)\n logger.reason(f\"Extracted {len(rows)} data row(s)\")\n\n # Debug: log first row keys and translation column value\n if rows:\n first_row = rows[0]\n logger.reason(\n f\"Row keys={list(first_row.keys())} \"\n f\"target_col={job.translation_column} \"\n f\"val='{first_row.get(job.translation_column, '')}'\"\n )\n\n return rows\n # #endregion _fetch_sample_rows\n" + "body": " # #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample]\n # @BRIEF Fetch sample rows from the Superset dataset for preview.\n # @PRE job has source_datasource_id and translation_column.\n # @POST Returns list of dicts with row data from Superset chart data API.\n # @SIDE_EFFECT Calls Superset chart data endpoint with pagination.\n def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:\n with belief_scope(\"TranslationPreview._fetch_sample_rows\"):\n # Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)\n environments = self.config_manager.get_environments()\n target_env_id = env_id or job.environment_id or job.source_dialect or \"\"\n env_config = next(\n (e for e in environments if e.id == target_env_id),\n None,\n )\n if not env_config:\n log.explore(\"Could not find environment for datasource\", error=\"Environment not found for datasource\",\n payload={\n \"env_id\": target_env_id,\n })\n # Fallback: try first environment\n if environments:\n env_config = environments[0]\n log.explore(\"Falling back to first available environment\", error=\"Environment fallback used\",\n payload={\n \"env_id\": env_config.id,\n })\n else:\n raise ValueError(\"No Superset environments configured\")\n\n client = SupersetClient(env_config)\n\n # Fetch dataset detail to build proper query context\n dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))\n\n # Build query context for chart data endpoint.\n # Virtual columns (e.g. comment_text_ru) are NOT resolved when:\n # - result_type=\"query\" (physical columns only)\n # - query_mode=\"raw\" (virtual columns unavailable in raw mode)\n # Solution: remove both result_type=\"query\" AND query_mode=\"raw\",\n # use aggregate mode with no metrics — this resolves virtual columns.\n query_context = client.build_dataset_preview_query_context(\n dataset_id=int(job.source_datasource_id),\n dataset_record=dataset_detail,\n template_params={},\n effective_filters=[],\n )\n\n # Modify: use result_type=\"samples\" which returns sample data\n # including all columns (physical + virtual), without needing\n # explicit column objects that trigger validation errors.\n queries = query_context.get(\"queries\", [])\n if queries:\n queries[0][\"row_limit\"] = sample_size\n queries[0].pop(\"result_type\", None)\n queries[0].pop(\"columns\", None)\n queries[0][\"metrics\"] = []\n query_context[\"result_type\"] = \"samples\"\n form_data = query_context.get(\"form_data\", {})\n form_data.pop(\"query_mode\", None)\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/chart/data\",\n data=json.dumps(query_context),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n log.explore(\"Chart data API failed\", error=str(e))\n raise ValueError(f\"Failed to fetch sample data from Superset: {e}\")\n\n # Parse response\n rows = self._extract_data_rows(response)\n log.reason(f\"Extracted {len(rows)} data row(s)\")\n\n # Debug: log first row keys and translation column value\n if rows:\n first_row = rows[0]\n log.reason(\n f\"Row keys={list(first_row.keys())} \"\n f\"target_col={job.translation_column} \"\n f\"val='{first_row.get(job.translation_column, '')}'\"\n )\n\n return rows\n # #endregion _fetch_sample_rows\n" }, { "contract_id": "_extract_data_rows", "contract_type": "Function", "file_path": "backend/src/plugins/translate/preview.py", - "start_line": 594, - "end_line": 625, + "start_line": 599, + "end_line": 630, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -65831,7 +66840,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/translate/scheduler.py", "start_line": 1, - "end_line": 365, + "end_line": 366, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -65890,14 +66899,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron]\n# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationSchedule]\n# @RELATION DEPENDS_ON -> [SchedulerService]\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @PRE Database session and SchedulerService are available.\n# @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule.\n# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.\n# @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule]\n# @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time.\n# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.\n# @REJECTED Separate scheduler instance would create resource contention.\n# @REJECTED Polling-based approach — event-driven APScheduler is more precise.\n\nimport uuid\nfrom datetime import datetime, timezone, timedelta\nfrom typing import Any, Dict, List, Optional\n\nfrom sqlalchemy.orm import Session\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.config_manager import ConfigManager\nfrom ...models.translate import TranslationSchedule, TranslationJob, TranslationRun\nfrom .events import TranslationEventLog\n\n\n# #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud]\n# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.\n# @PRE Database session and ConfigManager are available.\n# @POST Schedule rows created/updated/deleted with event logging.\n# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events.\n# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule]\n# @INVARIANT Exactly one schedule per job (upsert pattern).\nclass TranslationScheduler:\n\n def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n self.event_log = TranslationEventLog(db)\n\n # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]\n # @BRIEF Create a new schedule for a job with cron expression and event logging.\n # @PRE job_id exists. cron_expression is valid.\n # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def create_schedule(\n self,\n job_id: str,\n cron_expression: str,\n timezone: str = \"UTC\",\n is_active: bool = True,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.create_schedule\"):\n # Verify job exists\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n\n schedule = TranslationSchedule(\n id=str(uuid.uuid4()),\n job_id=job_id,\n cron_expression=cron_expression,\n timezone=timezone,\n is_active=is_active,\n created_by=self.current_user,\n )\n self.db.add(schedule)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_CREATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": cron_expression,\n \"timezone\": timezone,\n },\n created_by=self.current_user,\n )\n\n logger.reflect(\"Schedule created\", {\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion create_schedule\n\n # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]\n # @BRIEF Update an existing schedule's cron expression, timezone, or active state.\n # @PRE job_id has an existing schedule.\n # @POST Schedule updated; SCHEDULE_UPDATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def update_schedule(\n self,\n job_id: str,\n cron_expression: Optional[str] = None,\n timezone_str: Optional[str] = None,\n is_active: Optional[bool] = None,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.update_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n if cron_expression is not None:\n schedule.cron_expression = cron_expression\n if timezone_str is not None:\n schedule.timezone = timezone_str\n if is_active is not None:\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_UPDATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n },\n created_by=self.current_user,\n )\n\n logger.reflect(\"Schedule updated\", {\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion update_schedule\n\n # #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete]\n # @BRIEF Delete a schedule for a job with event logging.\n # @PRE job_id has an existing schedule.\n # @POST Schedule deleted; SCHEDULE_DELETED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def delete_schedule(self, job_id: str) -> None:\n with belief_scope(\"TranslationScheduler.delete_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n schedule_id = schedule.id\n self.db.delete(schedule)\n self.db.commit()\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_DELETED\",\n payload={\"schedule_id\": schedule_id},\n created_by=self.current_user,\n )\n\n logger.reflect(\"Schedule deleted\", {\"schedule_id\": schedule_id, \"job_id\": job_id})\n # #endregion delete_schedule\n\n # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]\n # @BRIEF Enable or disable a schedule by setting is_active flag.\n # @PRE job_id has an existing schedule.\n # @POST Schedule is_active updated.\n # @SIDE_EFFECT DB write.\n def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.set_schedule_active\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n logger.reflect(\"Schedule active state set\", {\n \"schedule_id\": schedule.id,\n \"job_id\": job_id,\n \"is_active\": is_active,\n })\n return schedule\n # #endregion set_schedule_active\n\n # #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get]\n # @BRIEF Get the schedule for a job by job_id.\n def get_schedule(self, job_id: str) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.get_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n return schedule\n # #endregion get_schedule\n\n # #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list]\n # @BRIEF List all active schedules.\n @staticmethod\n def list_active_schedules(db: Session) -> List[TranslationSchedule]:\n return (\n db.query(TranslationSchedule)\n .filter(TranslationSchedule.is_active == True)\n .all()\n )\n # #endregion list_active_schedules\n\n # #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next]\n # @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger.\n def get_next_executions(cron_expression: str, timezone_str: str = \"UTC\", n: int = 3) -> List[str]:\n from zoneinfo import ZoneInfo\n from apscheduler.triggers.cron import CronTrigger\n from apscheduler.triggers.interval import IntervalTrigger\n\n try:\n tz = ZoneInfo(timezone_str)\n trigger = CronTrigger.from_crontab(cron_expression, timezone=tz)\n except (ValueError, KeyError) as e:\n logger.warning(f\"[get_next_executions] Invalid cron: {e}\")\n return []\n\n now = datetime.now(tz)\n results = []\n next_time = now\n prev = None\n for _ in range(n):\n ft = trigger.get_next_fire_time(prev, next_time)\n if ft is None:\n break\n results.append(ft.isoformat())\n prev = ft\n next_time = ft\n return results\n # #endregion get_next_executions\n\n\n# #endregion TranslationScheduler\n\n\n# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]\n# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.\n# @PRE schedule_id is valid and job exists.\n# @POST Translation run created and executed if no concurrent run exists.\n# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.\n# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]\n# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.\n# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.\ndef execute_scheduled_translation(\n schedule_id: str,\n job_id: str,\n db_session_maker,\n config_manager: ConfigManager,\n) -> None:\n \"\"\"APScheduler job callback for scheduled translations.\"\"\"\n db: Session = db_session_maker()\n try:\n with belief_scope(\"execute_scheduled_translation\"):\n # Load schedule\n schedule = db.query(TranslationSchedule).filter(\n TranslationSchedule.id == schedule_id,\n TranslationSchedule.job_id == job_id,\n ).first()\n if not schedule or not schedule.is_active:\n logger.info(f\"[scheduled_translation] Schedule inactive/missing: {schedule_id}\")\n return\n\n logger.reason(\"Scheduled translation triggered\", {\n \"schedule_id\": schedule_id,\n \"job_id\": job_id,\n })\n\n # Concurrency check: max 1 pending/running run per job\n active_run = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.status.in_([\"PENDING\", \"RUNNING\"]),\n )\n .first()\n )\n if active_run:\n logger.explore(\"Skipping scheduled run — concurrent run in progress\", {\n \"job_id\": job_id,\n \"existing_run_id\": active_run.id,\n })\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"skipped_concurrent\", \"existing_run_id\": active_run.id},\n )\n return\n\n # Check baseline expiry\n baseline_expired = False\n most_recent = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.insert_status == \"succeeded\",\n )\n .order_by(TranslationRun.created_at.desc())\n .first()\n )\n if most_recent:\n age = datetime.now(timezone.utc) - most_recent.created_at\n if age > timedelta(days=90):\n baseline_expired = True\n logger.reason(\"Baseline expired — full translation\", {\n \"job_id\": job_id,\n \"last_run\": most_recent.created_at.isoformat(),\n \"age_days\": age.days,\n })\n\n # Import and execute\n from .orchestrator import TranslationOrchestrator\n\n orch = TranslationOrchestrator(db, config_manager, current_user=\"scheduler\")\n run = orch.start_run(\n job_id=job_id,\n is_scheduled=True,\n )\n run.trigger_type = \"scheduled\"\n db.flush()\n\n if baseline_expired:\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"baseline_expired\", \"age_days\": age.days},\n )\n\n # Execute in same thread (APScheduler runs in background thread pool)\n try:\n orch.execute_run(run)\n run.insert_status = run.insert_status or \"succeeded\"\n except Exception as exec_err:\n logger.explore(\"Scheduled translation execution failed\", {\n \"run_id\": run.id,\n \"error\": str(exec_err),\n })\n # Leave schedule enabled — schedule continues on failure\n run.status = \"FAILED\"\n run.error_message = str(exec_err)\n run.completed_at = datetime.now(timezone.utc)\n\n # Update schedule tracking\n schedule.last_run_at = datetime.now(timezone.utc)\n db.commit()\n\n logger.reflect(\"Scheduled translation complete\", {\n \"schedule_id\": schedule_id,\n \"run_id\": run.id,\n \"status\": run.status,\n })\n except Exception as e:\n logger.error(f\"[scheduled_translation] Unexpected error: {e}\")\n finally:\n db.close()\n# #endregion execute_scheduled_translation\n# #endregion TranslationScheduler\n" + "body": "# #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron]\n# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationSchedule]\n# @RELATION DEPENDS_ON -> [SchedulerService]\n# @RELATION DEPENDS_ON -> [TranslationOrchestrator]\n# @RELATION DEPENDS_ON -> [TranslationEventLog]\n# @PRE Database session and SchedulerService are available.\n# @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule.\n# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.\n# @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule]\n# @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time.\n# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.\n# @REJECTED Separate scheduler instance would create resource contention.\n# @REJECTED Polling-based approach — event-driven APScheduler is more precise.\n\nimport uuid\nfrom datetime import datetime, timezone, timedelta\nfrom typing import Any, Dict, List, Optional\n\nfrom sqlalchemy.orm import Session\n\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.logger import belief_scope\nfrom ...core.config_manager import ConfigManager\nfrom ...models.translate import TranslationSchedule, TranslationJob, TranslationRun\nfrom .events import TranslationEventLog\n\nlog = MarkerLogger(\"TranslationScheduler\")\n\n\n# #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud]\n# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.\n# @PRE Database session and ConfigManager are available.\n# @POST Schedule rows created/updated/deleted with event logging.\n# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events.\n# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule]\n# @INVARIANT Exactly one schedule per job (upsert pattern).\nclass TranslationScheduler:\n\n def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n self.event_log = TranslationEventLog(db)\n\n # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]\n # @BRIEF Create a new schedule for a job with cron expression and event logging.\n # @PRE job_id exists. cron_expression is valid.\n # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def create_schedule(\n self,\n job_id: str,\n cron_expression: str,\n timezone: str = \"UTC\",\n is_active: bool = True,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.create_schedule\"):\n # Verify job exists\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n\n schedule = TranslationSchedule(\n id=str(uuid.uuid4()),\n job_id=job_id,\n cron_expression=cron_expression,\n timezone=timezone,\n is_active=is_active,\n created_by=self.current_user,\n )\n self.db.add(schedule)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_CREATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": cron_expression,\n \"timezone\": timezone,\n },\n created_by=self.current_user,\n )\n\n log.reflect(\"Schedule created\", payload={\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion create_schedule\n\n # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]\n # @BRIEF Update an existing schedule's cron expression, timezone, or active state.\n # @PRE job_id has an existing schedule.\n # @POST Schedule updated; SCHEDULE_UPDATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def update_schedule(\n self,\n job_id: str,\n cron_expression: Optional[str] = None,\n timezone_str: Optional[str] = None,\n is_active: Optional[bool] = None,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.update_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n if cron_expression is not None:\n schedule.cron_expression = cron_expression\n if timezone_str is not None:\n schedule.timezone = timezone_str\n if is_active is not None:\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_UPDATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n },\n created_by=self.current_user,\n )\n\n log.reflect(\"Schedule updated\", payload={\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion update_schedule\n\n # #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete]\n # @BRIEF Delete a schedule for a job with event logging.\n # @PRE job_id has an existing schedule.\n # @POST Schedule deleted; SCHEDULE_DELETED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def delete_schedule(self, job_id: str) -> None:\n with belief_scope(\"TranslationScheduler.delete_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n schedule_id = schedule.id\n self.db.delete(schedule)\n self.db.commit()\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_DELETED\",\n payload={\"schedule_id\": schedule_id},\n created_by=self.current_user,\n )\n\n log.reflect(\"Schedule deleted\", payload={\"schedule_id\": schedule_id, \"job_id\": job_id})\n # #endregion delete_schedule\n\n # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]\n # @BRIEF Enable or disable a schedule by setting is_active flag.\n # @PRE job_id has an existing schedule.\n # @POST Schedule is_active updated.\n # @SIDE_EFFECT DB write.\n def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.set_schedule_active\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n log.reflect(\"Schedule active state set\", payload={\n \"schedule_id\": schedule.id,\n \"job_id\": job_id,\n \"is_active\": is_active,\n })\n return schedule\n # #endregion set_schedule_active\n\n # #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get]\n # @BRIEF Get the schedule for a job by job_id.\n def get_schedule(self, job_id: str) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.get_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n return schedule\n # #endregion get_schedule\n\n # #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list]\n # @BRIEF List all active schedules.\n @staticmethod\n def list_active_schedules(db: Session) -> List[TranslationSchedule]:\n return (\n db.query(TranslationSchedule)\n .filter(TranslationSchedule.is_active == True)\n .all()\n )\n # #endregion list_active_schedules\n\n # #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next]\n # @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger.\n def get_next_executions(cron_expression: str, timezone_str: str = \"UTC\", n: int = 3) -> List[str]:\n from zoneinfo import ZoneInfo\n from apscheduler.triggers.cron import CronTrigger\n from apscheduler.triggers.interval import IntervalTrigger\n\n try:\n tz = ZoneInfo(timezone_str)\n trigger = CronTrigger.from_crontab(cron_expression, timezone=tz)\n except (ValueError, KeyError) as e:\n log.explore(\"Invalid cron\", error=str(e))\n return []\n\n now = datetime.now(tz)\n results = []\n next_time = now\n prev = None\n for _ in range(n):\n ft = trigger.get_next_fire_time(prev, next_time)\n if ft is None:\n break\n results.append(ft.isoformat())\n prev = ft\n next_time = ft\n return results\n # #endregion get_next_executions\n\n\n# #endregion TranslationScheduler\n\n\n# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]\n# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.\n# @PRE schedule_id is valid and job exists.\n# @POST Translation run created and executed if no concurrent run exists.\n# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.\n# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]\n# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.\n# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.\ndef execute_scheduled_translation(\n schedule_id: str,\n job_id: str,\n db_session_maker,\n config_manager: ConfigManager,\n) -> None:\n \"\"\"APScheduler job callback for scheduled translations.\"\"\"\n db: Session = db_session_maker()\n try:\n with belief_scope(\"execute_scheduled_translation\"):\n # Load schedule\n schedule = db.query(TranslationSchedule).filter(\n TranslationSchedule.id == schedule_id,\n TranslationSchedule.job_id == job_id,\n ).first()\n if not schedule or not schedule.is_active:\n log.reason(\"Schedule inactive/missing\", payload={\"schedule_id\": schedule_id})\n return\n\n log.reason(\"Scheduled translation triggered\", payload={\n \"schedule_id\": schedule_id,\n \"job_id\": job_id,\n })\n\n # Concurrency check: max 1 pending/running run per job\n active_run = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.status.in_([\"PENDING\", \"RUNNING\"]),\n )\n .first()\n )\n if active_run:\n log.explore(\"Skipping scheduled run — concurrent run in progress\", error=\"Concurrent run detected\",\n payload={\n \"job_id\": job_id,\n \"existing_run_id\": active_run.id,\n })\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"skipped_concurrent\", \"existing_run_id\": active_run.id},\n )\n return\n\n # Check baseline expiry\n baseline_expired = False\n most_recent = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.insert_status == \"succeeded\",\n )\n .order_by(TranslationRun.created_at.desc())\n .first()\n )\n if most_recent:\n age = datetime.now(timezone.utc) - most_recent.created_at\n if age > timedelta(days=90):\n baseline_expired = True\n log.reason(\"Baseline expired — full translation\", payload={\n \"job_id\": job_id,\n \"last_run\": most_recent.created_at.isoformat(),\n \"age_days\": age.days,\n })\n\n # Import and execute\n from .orchestrator import TranslationOrchestrator\n\n orch = TranslationOrchestrator(db, config_manager, current_user=\"scheduler\")\n run = orch.start_run(\n job_id=job_id,\n is_scheduled=True,\n )\n run.trigger_type = \"scheduled\"\n db.flush()\n\n if baseline_expired:\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"baseline_expired\", \"age_days\": age.days},\n )\n\n # Execute in same thread (APScheduler runs in background thread pool)\n try:\n orch.execute_run(run)\n run.insert_status = run.insert_status or \"succeeded\"\n except Exception as exec_err:\n log.explore(\"Scheduled translation execution failed\", error=str(exec_err))\n # Leave schedule enabled — schedule continues on failure\n run.status = \"FAILED\"\n run.error_message = str(exec_err)\n run.completed_at = datetime.now(timezone.utc)\n\n # Update schedule tracking\n schedule.last_run_at = datetime.now(timezone.utc)\n db.commit()\n\n log.reflect(\"Scheduled translation complete\", payload={\n \"schedule_id\": schedule_id,\n \"run_id\": run.id,\n \"status\": run.status,\n })\n except Exception as e:\n log.explore(\"Unexpected error\", error=str(e))\n finally:\n db.close()\n# #endregion execute_scheduled_translation\n# #endregion TranslationScheduler\n" }, { "contract_id": "create_schedule", "contract_type": "Function", "file_path": "backend/src/plugins/translate/scheduler.py", - "start_line": 44, - "end_line": 87, + "start_line": 47, + "end_line": 90, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -65935,14 +66944,14 @@ } ], "anchor_syntax": "region", - "body": " # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]\n # @BRIEF Create a new schedule for a job with cron expression and event logging.\n # @PRE job_id exists. cron_expression is valid.\n # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def create_schedule(\n self,\n job_id: str,\n cron_expression: str,\n timezone: str = \"UTC\",\n is_active: bool = True,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.create_schedule\"):\n # Verify job exists\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n\n schedule = TranslationSchedule(\n id=str(uuid.uuid4()),\n job_id=job_id,\n cron_expression=cron_expression,\n timezone=timezone,\n is_active=is_active,\n created_by=self.current_user,\n )\n self.db.add(schedule)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_CREATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": cron_expression,\n \"timezone\": timezone,\n },\n created_by=self.current_user,\n )\n\n logger.reflect(\"Schedule created\", {\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion create_schedule\n" + "body": " # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create]\n # @BRIEF Create a new schedule for a job with cron expression and event logging.\n # @PRE job_id exists. cron_expression is valid.\n # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def create_schedule(\n self,\n job_id: str,\n cron_expression: str,\n timezone: str = \"UTC\",\n is_active: bool = True,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.create_schedule\"):\n # Verify job exists\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n\n schedule = TranslationSchedule(\n id=str(uuid.uuid4()),\n job_id=job_id,\n cron_expression=cron_expression,\n timezone=timezone,\n is_active=is_active,\n created_by=self.current_user,\n )\n self.db.add(schedule)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_CREATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": cron_expression,\n \"timezone\": timezone,\n },\n created_by=self.current_user,\n )\n\n log.reflect(\"Schedule created\", payload={\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion create_schedule\n" }, { "contract_id": "update_schedule", "contract_type": "Function", "file_path": "backend/src/plugins/translate/scheduler.py", - "start_line": 89, - "end_line": 132, + "start_line": 92, + "end_line": 135, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -65980,14 +66989,14 @@ } ], "anchor_syntax": "region", - "body": " # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]\n # @BRIEF Update an existing schedule's cron expression, timezone, or active state.\n # @PRE job_id has an existing schedule.\n # @POST Schedule updated; SCHEDULE_UPDATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def update_schedule(\n self,\n job_id: str,\n cron_expression: Optional[str] = None,\n timezone_str: Optional[str] = None,\n is_active: Optional[bool] = None,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.update_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n if cron_expression is not None:\n schedule.cron_expression = cron_expression\n if timezone_str is not None:\n schedule.timezone = timezone_str\n if is_active is not None:\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_UPDATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n },\n created_by=self.current_user,\n )\n\n logger.reflect(\"Schedule updated\", {\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion update_schedule\n" + "body": " # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update]\n # @BRIEF Update an existing schedule's cron expression, timezone, or active state.\n # @PRE job_id has an existing schedule.\n # @POST Schedule updated; SCHEDULE_UPDATED event logged.\n # @SIDE_EFFECT DB write; event logging.\n def update_schedule(\n self,\n job_id: str,\n cron_expression: Optional[str] = None,\n timezone_str: Optional[str] = None,\n is_active: Optional[bool] = None,\n ) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.update_schedule\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n if cron_expression is not None:\n schedule.cron_expression = cron_expression\n if timezone_str is not None:\n schedule.timezone = timezone_str\n if is_active is not None:\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n self.event_log.log_event(\n job_id=job_id,\n event_type=\"SCHEDULE_UPDATED\",\n payload={\n \"schedule_id\": schedule.id,\n \"cron_expression\": schedule.cron_expression,\n \"timezone\": schedule.timezone,\n \"is_active\": schedule.is_active,\n },\n created_by=self.current_user,\n )\n\n log.reflect(\"Schedule updated\", payload={\"schedule_id\": schedule.id, \"job_id\": job_id})\n return schedule\n # #endregion update_schedule\n" }, { "contract_id": "set_schedule_active", "contract_type": "Function", "file_path": "backend/src/plugins/translate/scheduler.py", - "start_line": 161, - "end_line": 185, + "start_line": 164, + "end_line": 188, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66025,14 +67034,14 @@ } ], "anchor_syntax": "region", - "body": " # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]\n # @BRIEF Enable or disable a schedule by setting is_active flag.\n # @PRE job_id has an existing schedule.\n # @POST Schedule is_active updated.\n # @SIDE_EFFECT DB write.\n def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.set_schedule_active\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n logger.reflect(\"Schedule active state set\", {\n \"schedule_id\": schedule.id,\n \"job_id\": job_id,\n \"is_active\": is_active,\n })\n return schedule\n # #endregion set_schedule_active\n" + "body": " # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate]\n # @BRIEF Enable or disable a schedule by setting is_active flag.\n # @PRE job_id has an existing schedule.\n # @POST Schedule is_active updated.\n # @SIDE_EFFECT DB write.\n def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:\n with belief_scope(\"TranslationScheduler.set_schedule_active\"):\n schedule = self.db.query(TranslationSchedule).filter(\n TranslationSchedule.job_id == job_id\n ).first()\n if not schedule:\n raise ValueError(f\"No schedule found for job '{job_id}'\")\n\n schedule.is_active = is_active\n schedule.updated_at = datetime.now(timezone.utc)\n self.db.commit()\n self.db.refresh(schedule)\n\n log.reflect(\"Schedule active state set\", payload={\n \"schedule_id\": schedule.id,\n \"job_id\": job_id,\n \"is_active\": is_active,\n })\n return schedule\n # #endregion set_schedule_active\n" }, { "contract_id": "list_active_schedules", "contract_type": "Function", "file_path": "backend/src/plugins/translate/scheduler.py", - "start_line": 199, - "end_line": 208, + "start_line": 202, + "end_line": 211, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66064,8 +67073,8 @@ "contract_id": "execute_scheduled_translation", "contract_type": "Function", "file_path": "backend/src/plugins/translate/scheduler.py", - "start_line": 242, - "end_line": 364, + "start_line": 245, + "end_line": 365, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -66106,14 +67115,14 @@ } ], "anchor_syntax": "region", - "body": "# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]\n# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.\n# @PRE schedule_id is valid and job exists.\n# @POST Translation run created and executed if no concurrent run exists.\n# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.\n# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]\n# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.\n# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.\ndef execute_scheduled_translation(\n schedule_id: str,\n job_id: str,\n db_session_maker,\n config_manager: ConfigManager,\n) -> None:\n \"\"\"APScheduler job callback for scheduled translations.\"\"\"\n db: Session = db_session_maker()\n try:\n with belief_scope(\"execute_scheduled_translation\"):\n # Load schedule\n schedule = db.query(TranslationSchedule).filter(\n TranslationSchedule.id == schedule_id,\n TranslationSchedule.job_id == job_id,\n ).first()\n if not schedule or not schedule.is_active:\n logger.info(f\"[scheduled_translation] Schedule inactive/missing: {schedule_id}\")\n return\n\n logger.reason(\"Scheduled translation triggered\", {\n \"schedule_id\": schedule_id,\n \"job_id\": job_id,\n })\n\n # Concurrency check: max 1 pending/running run per job\n active_run = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.status.in_([\"PENDING\", \"RUNNING\"]),\n )\n .first()\n )\n if active_run:\n logger.explore(\"Skipping scheduled run — concurrent run in progress\", {\n \"job_id\": job_id,\n \"existing_run_id\": active_run.id,\n })\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"skipped_concurrent\", \"existing_run_id\": active_run.id},\n )\n return\n\n # Check baseline expiry\n baseline_expired = False\n most_recent = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.insert_status == \"succeeded\",\n )\n .order_by(TranslationRun.created_at.desc())\n .first()\n )\n if most_recent:\n age = datetime.now(timezone.utc) - most_recent.created_at\n if age > timedelta(days=90):\n baseline_expired = True\n logger.reason(\"Baseline expired — full translation\", {\n \"job_id\": job_id,\n \"last_run\": most_recent.created_at.isoformat(),\n \"age_days\": age.days,\n })\n\n # Import and execute\n from .orchestrator import TranslationOrchestrator\n\n orch = TranslationOrchestrator(db, config_manager, current_user=\"scheduler\")\n run = orch.start_run(\n job_id=job_id,\n is_scheduled=True,\n )\n run.trigger_type = \"scheduled\"\n db.flush()\n\n if baseline_expired:\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"baseline_expired\", \"age_days\": age.days},\n )\n\n # Execute in same thread (APScheduler runs in background thread pool)\n try:\n orch.execute_run(run)\n run.insert_status = run.insert_status or \"succeeded\"\n except Exception as exec_err:\n logger.explore(\"Scheduled translation execution failed\", {\n \"run_id\": run.id,\n \"error\": str(exec_err),\n })\n # Leave schedule enabled — schedule continues on failure\n run.status = \"FAILED\"\n run.error_message = str(exec_err)\n run.completed_at = datetime.now(timezone.utc)\n\n # Update schedule tracking\n schedule.last_run_at = datetime.now(timezone.utc)\n db.commit()\n\n logger.reflect(\"Scheduled translation complete\", {\n \"schedule_id\": schedule_id,\n \"run_id\": run.id,\n \"status\": run.status,\n })\n except Exception as e:\n logger.error(f\"[scheduled_translation] Unexpected error: {e}\")\n finally:\n db.close()\n# #endregion execute_scheduled_translation\n" + "body": "# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute]\n# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection.\n# @PRE schedule_id is valid and job exists.\n# @POST Translation run created and executed if no concurrent run exists.\n# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging.\n# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None]\n# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time.\n# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.\ndef execute_scheduled_translation(\n schedule_id: str,\n job_id: str,\n db_session_maker,\n config_manager: ConfigManager,\n) -> None:\n \"\"\"APScheduler job callback for scheduled translations.\"\"\"\n db: Session = db_session_maker()\n try:\n with belief_scope(\"execute_scheduled_translation\"):\n # Load schedule\n schedule = db.query(TranslationSchedule).filter(\n TranslationSchedule.id == schedule_id,\n TranslationSchedule.job_id == job_id,\n ).first()\n if not schedule or not schedule.is_active:\n log.reason(\"Schedule inactive/missing\", payload={\"schedule_id\": schedule_id})\n return\n\n log.reason(\"Scheduled translation triggered\", payload={\n \"schedule_id\": schedule_id,\n \"job_id\": job_id,\n })\n\n # Concurrency check: max 1 pending/running run per job\n active_run = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.status.in_([\"PENDING\", \"RUNNING\"]),\n )\n .first()\n )\n if active_run:\n log.explore(\"Skipping scheduled run — concurrent run in progress\", error=\"Concurrent run detected\",\n payload={\n \"job_id\": job_id,\n \"existing_run_id\": active_run.id,\n })\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"skipped_concurrent\", \"existing_run_id\": active_run.id},\n )\n return\n\n # Check baseline expiry\n baseline_expired = False\n most_recent = (\n db.query(TranslationRun)\n .filter(\n TranslationRun.job_id == job_id,\n TranslationRun.insert_status == \"succeeded\",\n )\n .order_by(TranslationRun.created_at.desc())\n .first()\n )\n if most_recent:\n age = datetime.now(timezone.utc) - most_recent.created_at\n if age > timedelta(days=90):\n baseline_expired = True\n log.reason(\"Baseline expired — full translation\", payload={\n \"job_id\": job_id,\n \"last_run\": most_recent.created_at.isoformat(),\n \"age_days\": age.days,\n })\n\n # Import and execute\n from .orchestrator import TranslationOrchestrator\n\n orch = TranslationOrchestrator(db, config_manager, current_user=\"scheduler\")\n run = orch.start_run(\n job_id=job_id,\n is_scheduled=True,\n )\n run.trigger_type = \"scheduled\"\n db.flush()\n\n if baseline_expired:\n event_log = TranslationEventLog(db)\n event_log.log_event(\n job_id=job_id,\n run_id=run.id,\n event_type=\"RUN_STARTED\",\n payload={\"reason\": \"baseline_expired\", \"age_days\": age.days},\n )\n\n # Execute in same thread (APScheduler runs in background thread pool)\n try:\n orch.execute_run(run)\n run.insert_status = run.insert_status or \"succeeded\"\n except Exception as exec_err:\n log.explore(\"Scheduled translation execution failed\", error=str(exec_err))\n # Leave schedule enabled — schedule continues on failure\n run.status = \"FAILED\"\n run.error_message = str(exec_err)\n run.completed_at = datetime.now(timezone.utc)\n\n # Update schedule tracking\n schedule.last_run_at = datetime.now(timezone.utc)\n db.commit()\n\n log.reflect(\"Scheduled translation complete\", payload={\n \"schedule_id\": schedule_id,\n \"run_id\": run.id,\n \"status\": run.status,\n })\n except Exception as e:\n log.explore(\"Unexpected error\", error=str(e))\n finally:\n db.close()\n# #endregion execute_scheduled_translation\n" }, { "contract_id": "TranslateJobService", "contract_type": "Module", "file_path": "backend/src/plugins/translate/service.py", "start_line": 1, - "end_line": 576, + "end_line": 579, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -66166,14 +67175,14 @@ } ], "anchor_syntax": "region", - "body": "# #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation]\n# @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @PRE Database session and config manager are available.\n# @POST Translation jobs are created/updated/deleted with column validation and dialect caching.\n# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.\n# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse]\n# @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only.\n# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.\n# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.\n\nfrom typing import Any, Dict, List, Optional, Tuple\nfrom sqlalchemy.orm import Session\nfrom datetime import datetime, timezone\nimport uuid\n\nfrom ...core.logger import logger\nfrom ...core.config_manager import ConfigManager\nfrom ...core.superset_client import SupersetClient\nfrom ...models.translate import TranslationJob, TranslationJobDictionary\nfrom ...schemas.translate import (\n TranslateJobCreate,\n TranslateJobUpdate,\n TranslateJobResponse,\n DatasourceColumnsResponse,\n DatasourceColumnResponse,\n)\n\n# Supported database dialects for translation\nSUPPORTED_DIALECTS = {\n \"postgresql\", \"mysql\", \"clickhouse\", \"sqlite\", \"mssql\",\n \"oracle\", \"snowflake\", \"bigquery\", \"redshift\", \"presto\",\n \"trino\", \"druid\", \"hive\", \"spark\", \"databricks\",\n}\n\n\n# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]\n# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.\n# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.\n# @POST Returns normalized dialect string or raises ValueError if unsupported.\n# @SIDE_EFFECT None — pure data extraction.\ndef get_dialect_from_database(database_record: Dict[str, Any]) -> str:\n \"\"\"Extract and validate dialect from Superset database record.\"\"\"\n logger.reason(\"Extracting dialect from database record\", {\n \"has_backend\": bool(database_record.get(\"backend\") or database_record.get(\"engine\")),\n })\n backend = (\n database_record.get(\"backend\")\n or database_record.get(\"engine\")\n or \"\"\n ).lower().strip()\n\n if not backend:\n logger.explore(\"Could not determine database dialect\", {\n \"record_keys\": list(database_record.keys()),\n })\n raise ValueError(\"Could not determine database dialect from connection\")\n\n # Map Superset backend names to normalized dialect\n dialect_map = {\n \"postgresql\": \"postgresql\",\n \"greenplum\": \"postgresql\",\n \"mysql\": \"mysql\",\n \"clickhouse\": \"clickhouse\",\n \"clickhousedb\": \"clickhouse\",\n \"sqlite\": \"sqlite\",\n \"mssql\": \"mssql\",\n \"oracle\": \"oracle\",\n \"snowflake\": \"snowflake\",\n \"bigquery\": \"bigquery\",\n \"redshift\": \"redshift\",\n \"presto\": \"presto\",\n \"trino\": \"trino\",\n \"druid\": \"druid\",\n \"hive\": \"hive\",\n \"spark\": \"spark\",\n \"databricks\": \"databricks\",\n }\n\n normalized = dialect_map.get(backend, backend)\n if normalized not in SUPPORTED_DIALECTS:\n logger.explore(\"Unsupported dialect\", {\"backend\": backend, \"normalized\": normalized})\n raise ValueError(\n f\"Unsupported database dialect: '{backend}'. \"\n f\"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}\"\n )\n logger.reflect(\"Dialect validated\", {\"dialect\": normalized})\n return normalized\n# #endregion get_dialect_from_database\n\n\n# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]\n# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.\n# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.\n# @POST Returns (columns_list, dialect_string) or raises on failure.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\ndef fetch_datasource_metadata(\n dataset_id: int,\n env_id: str,\n config_manager: ConfigManager,\n) -> Tuple[List[Dict[str, Any]], str]:\n \"\"\"Fetch column metadata and database dialect for a datasource from Superset.\"\"\"\n logger.reason(\"Fetching datasource metadata\", {\n \"dataset_id\": dataset_id,\n \"env_id\": env_id,\n })\n # Find environment config\n environments = config_manager.get_environments()\n env_config = next((e for e in environments if e.id == env_id), None)\n if not env_config:\n logger.explore(\"Environment not found\", {\"env_id\": env_id})\n raise ValueError(f\"Superset environment '{env_id}' not found in configuration\")\n\n # Create Superset client and fetch dataset detail\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(dataset_id)\n\n # Extract columns\n raw_columns = dataset_detail.get(\"columns\", [])\n columns = []\n for col in raw_columns:\n col_name = col.get(\"name\") or col.get(\"column_name\")\n if not col_name:\n continue\n columns.append({\n \"name\": str(col_name),\n \"type\": col.get(\"type\"),\n \"is_physical\": col.get(\"is_physical\", True),\n \"is_dttm\": col.get(\"is_dttm\", False),\n \"description\": col.get(\"description\", \"\"),\n })\n\n # Extract database dialect from datasource\n database_info = dataset_detail.get(\"database\", {})\n if isinstance(database_info, dict):\n dialect = get_dialect_from_database(database_info)\n else:\n # Fallback: try to fetch database directly\n try:\n db_id = dataset_detail.get(\"database_id\")\n if db_id:\n db_record = client.get_database(int(db_id))\n db_result = db_record.get(\"result\", db_record) if isinstance(db_record, dict) else db_record\n dialect = get_dialect_from_database(db_result)\n else:\n raise ValueError(\"No database information available for this datasource\")\n except Exception as e:\n logger.explore(\"Could not fetch database dialect\", {\"error\": str(e)})\n raise ValueError(f\"Could not determine database dialect: {e}\")\n\n logger.reflect(\"Metadata fetched\", {\"columns\": len(columns), \"dialect\": dialect})\n return columns, dialect\n# #endregion fetch_datasource_metadata\n\n\n# #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns]\n# @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries.\ndef detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]:\n \"\"\"Return names of columns that are virtual (not physical).\"\"\"\n return [col[\"name\"] for col in columns if not col.get(\"is_physical\", True)]\n# #endregion detect_virtual_columns\n\n\n# #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud]\n# @BRIEF Service for translation job CRUD with validation and Superset integration.\n# @PRE Database session and config manager are available.\n# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate).\n# @SIDE_EFFECT Queries Superset for dialect detection and column validation.\n# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob]\n# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits.\nclass TranslateJobService:\n\n def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n\n # #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]\n # @BRIEF List translation jobs with optional status filter and pagination.\n def list_jobs(\n self,\n page: int = 1,\n page_size: int = 20,\n status_filter: Optional[str] = None,\n ) -> Tuple[int, List[TranslationJob]]:\n query = self.db.query(TranslationJob)\n\n if status_filter:\n query = query.filter(TranslationJob.status == status_filter)\n\n total = query.count()\n offset = (page - 1) * page_size\n jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()\n\n return total, jobs\n # #endregion list_jobs\n\n # #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]\n # @BRIEF Get a single translation job by ID, raising ValueError if not found.\n def get_job(self, job_id: str) -> TranslationJob:\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n return job\n # #endregion get_job\n\n # #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create]\n # @BRIEF Create a new translation job with column validation and dialect detection.\n # @PRE payload contains valid job configuration (name, upsert_strategy, etc).\n # @POST Returns the created TranslationJob with database_dialect cached.\n # @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect.\n def create_job(self, payload: TranslateJobCreate) -> TranslationJob:\n logger.reason(\"Creating translation job\", {\"name\": payload.name})\n\n # Validate: must have a translation column if datasource is configured\n if payload.source_datasource_id and not payload.translation_column:\n logger.explore(\"Missing translation column\", {\n \"source_datasource_id\": payload.source_datasource_id,\n })\n raise ValueError(\"A translation column is required when a datasource is selected\")\n\n # Validate upsert strategy\n valid_strategies = {\"MERGE\", \"INSERT\", \"UPDATE\"}\n if payload.upsert_strategy not in valid_strategies:\n raise ValueError(\n f\"Invalid upsert_strategy '{payload.upsert_strategy}'. \"\n f\"Must be one of: {', '.join(sorted(valid_strategies))}\"\n )\n\n # Detect database dialect and validate columns if datasource is specified\n dialect = payload.database_dialect\n if payload.source_datasource_id and (payload.environment_id or payload.source_dialect):\n # If no explicit dialect, try to detect it\n if not dialect:\n try:\n env_id = payload.environment_id or payload.source_dialect\n _, detected_dialect = fetch_datasource_metadata(\n int(payload.source_datasource_id),\n env_id,\n self.config_manager,\n )\n dialect = detected_dialect\n except Exception as e:\n logger.explore(\"Dialect detection failed\", {\"error\": str(e)})\n dialect = payload.source_dialect\n\n # Build job instance\n job = TranslationJob(\n id=str(uuid.uuid4()),\n name=payload.name,\n description=payload.description,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n database_dialect=dialect,\n source_datasource_id=payload.source_datasource_id,\n source_table=payload.source_table,\n target_schema=payload.target_schema,\n target_table=payload.target_table,\n source_key_cols=payload.source_key_cols or [],\n target_key_cols=payload.target_key_cols or [],\n translation_column=payload.translation_column,\n context_columns=payload.context_columns or [],\n target_language=payload.target_language,\n provider_id=payload.provider_id,\n batch_size=payload.batch_size,\n upsert_strategy=payload.upsert_strategy,\n environment_id=payload.environment_id,\n target_database_id=payload.target_database_id,\n status=\"DRAFT\",\n created_by=self.current_user,\n )\n self.db.add(job)\n self.db.flush()\n\n # Attach dictionaries\n if payload.dictionary_ids:\n for dict_id in payload.dictionary_ids:\n assoc = TranslationJobDictionary(\n id=str(uuid.uuid4()),\n job_id=job.id,\n dictionary_id=dict_id,\n )\n self.db.add(assoc)\n\n self.db.commit()\n self.db.refresh(job)\n logger.reflect(\"Job created\", {\"job_id\": job.id})\n return job\n # #endregion create_job\n\n # #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update]\n # @BRIEF Update an existing translation job with optional dialect re-detection.\n # @PRE payload contains fields to update; job_id exists.\n # @POST Returns the updated TranslationJob with re-detected dialect if datasource changed.\n # @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.\n def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:\n logger.reason(\"Updating translation job\", {\"job_id\": job_id})\n job = self.get_job(job_id)\n\n update_data = payload.model_dump(exclude_unset=True)\n dict_ids = update_data.pop(\"dictionary_ids\", None)\n\n for field, value in update_data.items():\n if hasattr(job, field):\n setattr(job, field, value)\n\n # Re-detect dialect if datasource changed\n if payload.source_datasource_id and not payload.database_dialect:\n try:\n env_id = (payload.environment_id or payload.source_dialect or job.environment_id or job.source_dialect)\n _, detected_dialect = fetch_datasource_metadata(\n int(payload.source_datasource_id),\n env_id,\n self.config_manager,\n )\n job.database_dialect = detected_dialect\n except Exception as e:\n logger.explore(\"Dialect re-detection failed\", {\"error\": str(e)})\n\n job.updated_at = datetime.now(timezone.utc)\n\n # Update dictionary associations if provided\n if dict_ids is not None:\n self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).delete()\n for dict_id in dict_ids:\n assoc = TranslationJobDictionary(\n id=str(uuid.uuid4()),\n job_id=job_id,\n dictionary_id=dict_id,\n )\n self.db.add(assoc)\n\n self.db.commit()\n self.db.refresh(job)\n logger.reflect(\"Job updated\", {\"job_id\": job_id})\n return job\n # #endregion update_job\n\n # #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete]\n # @BRIEF Delete a translation job and its dictionary associations.\n # @PRE job_id must exist.\n # @POST Job and all related TranslationJobDictionary records are deleted.\n # @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows.\n def delete_job(self, job_id: str) -> None:\n logger.reason(\"Deleting translation job\", {\"job_id\": job_id})\n job = self.get_job(job_id)\n\n # Delete dictionary associations\n self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).delete()\n\n self.db.delete(job)\n self.db.commit()\n logger.reflect(\"Job deleted\", {\"job_id\": job_id})\n # #endregion delete_job\n\n # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate]\n # @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations.\n # @PRE job_id must exist.\n # @POST Returns the duplicated TranslationJob with status DRAFT.\n # @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows.\n def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob:\n logger.reason(\"Duplicating translation job\", {\"job_id\": job_id, \"new_name\": new_name})\n source = self.get_job(job_id)\n\n # Copy all fields except id, created_at, updated_at, status\n new_job = TranslationJob(\n id=str(uuid.uuid4()),\n name=new_name or f\"{source.name} (Copy)\",\n description=source.description,\n source_dialect=source.source_dialect,\n target_dialect=source.target_dialect,\n database_dialect=source.database_dialect,\n source_datasource_id=source.source_datasource_id,\n source_table=source.source_table,\n target_schema=source.target_schema,\n target_table=source.target_table,\n source_key_cols=source.source_key_cols,\n target_key_cols=source.target_key_cols,\n translation_column=source.translation_column,\n context_columns=source.context_columns,\n target_language=source.target_language,\n provider_id=source.provider_id,\n batch_size=source.batch_size,\n upsert_strategy=source.upsert_strategy,\n environment_id=source.environment_id,\n target_database_id=source.target_database_id,\n status=\"DRAFT\",\n created_by=self.current_user,\n )\n self.db.add(new_job)\n self.db.flush()\n\n # Copy dictionary associations\n old_dicts = self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).all()\n for assoc in old_dicts:\n new_assoc = TranslationJobDictionary(\n id=str(uuid.uuid4()),\n job_id=new_job.id,\n dictionary_id=assoc.dictionary_id,\n )\n self.db.add(new_assoc)\n\n self.db.commit()\n self.db.refresh(new_job)\n logger.reflect(\"Job duplicated\", {\"new_job_id\": new_job.id, \"source_job_id\": job_id})\n return new_job\n # #endregion duplicate_job\n\n # #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries]\n def get_job_dictionary_ids(self, job_id: str) -> List[str]:\n assocs = self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).all()\n return [a.dictionary_id for a in assocs]\n # #endregion get_job_dictionary_ids\n\n # #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query]\n # @BRIEF List available Superset datasets for translation job creation with optional search filter.\n def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list:\n \"\"\"List Superset datasets available for translation.\"\"\"\n from ...core.superset_client import SupersetClient\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment '{env_id}' not found\")\n client = SupersetClient(env)\n _, datasets = client.get_datasets()\n result = []\n for ds in datasets:\n name = ds.get(\"table_name\", \"\")\n if search and search.lower() not in name.lower():\n continue\n db_info = ds.get(\"database\", {})\n backend = db_info.get(\"backend\", \"\")\n dialect = _extract_dialect(backend)\n result.append({\n \"id\": ds.get(\"id\"),\n \"table_name\": name,\n \"schema\": ds.get(\"schema\"),\n \"database_name\": db_info.get(\"database_name\", \"Unknown\"),\n \"database_dialect\": dialect,\n \"description\": ds.get(\"description\", \"\"),\n })\n return result\n # #endregion fetch_available_datasources\n# #endregion TranslateJobService\n\n\n# #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect]\ndef _extract_dialect(backend: str) -> str:\n \"\"\"Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...').\"\"\"\n if not backend:\n return \"unknown\"\n try:\n scheme = backend.split(\"://\")[0]\n dialect = scheme.split(\"+\")[0]\n return dialect.lower()\n except Exception:\n return \"unknown\"\n# #endregion _extract_dialect\n\n\n# #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization]\n# @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.\ndef job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse:\n return TranslateJobResponse(\n id=job.id,\n name=job.name,\n description=job.description,\n source_dialect=job.source_dialect,\n target_dialect=job.target_dialect,\n database_dialect=job.database_dialect,\n source_datasource_id=job.source_datasource_id,\n source_table=job.source_table,\n target_schema=job.target_schema,\n target_table=job.target_table,\n source_key_cols=job.source_key_cols or [],\n target_key_cols=job.target_key_cols or [],\n translation_column=job.translation_column,\n context_columns=job.context_columns or [],\n target_language=job.target_language,\n provider_id=job.provider_id,\n batch_size=job.batch_size or 50,\n upsert_strategy=job.upsert_strategy or \"MERGE\",\n status=job.status,\n created_by=job.created_by,\n created_at=job.created_at,\n updated_at=job.updated_at,\n dictionary_ids=dict_ids or [],\n environment_id=job.environment_id,\n target_database_id=job.target_database_id,\n )\n# #endregion job_to_response\n\n\n# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns]\n# @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse.\n# @PRE datasource_id is a valid Superset dataset ID.\n# @POST Returns DatasourceColumnsResponse with column metadata and database dialect.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\ndef get_datasource_columns(\n datasource_id: int,\n env_id: str,\n config_manager: ConfigManager,\n) -> DatasourceColumnsResponse:\n \"\"\"Fetch and return column metadata for a given Superset datasource.\"\"\"\n logger.reason(\"Fetching datasource columns\", {\"datasource_id\": datasource_id, \"env_id\": env_id})\n\n # Find environment config\n environments = config_manager.get_environments()\n env_config = next((e for e in environments if e.id == env_id), None)\n if not env_config:\n logger.explore(\"Environment not found\", {\"env_id\": env_id})\n raise ValueError(f\"Superset environment '{env_id}' not found\")\n\n # Create Superset client\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(datasource_id)\n\n # Extract database dialect\n database_info = dataset_detail.get(\"database\", {})\n dialect = None\n if isinstance(database_info, dict):\n try:\n dialect = get_dialect_from_database(database_info)\n except (ValueError, KeyError):\n dialect = None\n if dialect is None:\n database_id = dataset_detail.get(\"database_id\")\n if database_id:\n db_record = client.get_database(int(database_id))\n db_result = db_record.get(\"result\", db_record) if isinstance(db_record, dict) else db_record\n dialect = get_dialect_from_database(db_result)\n else:\n logger.explore(\"Could not determine dialect\", {\"datasource_id\": datasource_id})\n raise ValueError(\"Could not determine database dialect for this datasource\")\n\n # Extract columns\n raw_columns = dataset_detail.get(\"columns\", [])\n columns = []\n for col in raw_columns:\n col_name = col.get(\"name\") or col.get(\"column_name\")\n if not col_name:\n continue\n columns.append(DatasourceColumnResponse(\n name=str(col_name),\n type=col.get(\"type\"),\n is_physical=col.get(\"is_physical\", True),\n is_dttm=col.get(\"is_dttm\", False),\n description=col.get(\"description\", \"\"),\n ))\n\n result = DatasourceColumnsResponse(\n datasource_id=datasource_id,\n datasource_name=dataset_detail.get(\"table_name\"),\n schema_name=dataset_detail.get(\"schema\"),\n database_dialect=dialect,\n columns=columns,\n )\n logger.reflect(\"Datasource columns fetched\", {\n \"datasource_id\": datasource_id,\n \"columns_count\": len(columns),\n \"dialect\": dialect,\n })\n return result\n# #endregion get_datasource_columns\n\n# #endregion TranslateJobService\n" + "body": "# #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation]\n# @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @PRE Database session and config manager are available.\n# @POST Translation jobs are created/updated/deleted with column validation and dialect caching.\n# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.\n# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse]\n# @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only.\n# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.\n# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.\n\nfrom typing import Any, Dict, List, Optional, Tuple\nfrom sqlalchemy.orm import Session\nfrom datetime import datetime, timezone\nimport uuid\n\nfrom ...core.logger import logger\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.config_manager import ConfigManager\nfrom ...core.superset_client import SupersetClient\nfrom ...models.translate import TranslationJob, TranslationJobDictionary\nfrom ...schemas.translate import (\n TranslateJobCreate,\n TranslateJobUpdate,\n TranslateJobResponse,\n DatasourceColumnsResponse,\n DatasourceColumnResponse,\n)\n\nlog = MarkerLogger(\"TranslateJobService\")\n\n# Supported database dialects for translation\nSUPPORTED_DIALECTS = {\n \"postgresql\", \"mysql\", \"clickhouse\", \"sqlite\", \"mssql\",\n \"oracle\", \"snowflake\", \"bigquery\", \"redshift\", \"presto\",\n \"trino\", \"druid\", \"hive\", \"spark\", \"databricks\",\n}\n\n\n# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]\n# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.\n# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.\n# @POST Returns normalized dialect string or raises ValueError if unsupported.\n# @SIDE_EFFECT None — pure data extraction.\ndef get_dialect_from_database(database_record: Dict[str, Any]) -> str:\n \"\"\"Extract and validate dialect from Superset database record.\"\"\"\n log.reason(\"Extracting dialect from database record\", payload={\n \"has_backend\": bool(database_record.get(\"backend\") or database_record.get(\"engine\")),\n })\n backend = (\n database_record.get(\"backend\")\n or database_record.get(\"engine\")\n or \"\"\n ).lower().strip()\n\n if not backend:\n log.explore(\"Could not determine database dialect\", error=\"No backend/engine field in database record\", payload={\n \"record_keys\": list(database_record.keys()),\n })\n raise ValueError(\"Could not determine database dialect from connection\")\n\n # Map Superset backend names to normalized dialect\n dialect_map = {\n \"postgresql\": \"postgresql\",\n \"greenplum\": \"postgresql\",\n \"mysql\": \"mysql\",\n \"clickhouse\": \"clickhouse\",\n \"clickhousedb\": \"clickhouse\",\n \"sqlite\": \"sqlite\",\n \"mssql\": \"mssql\",\n \"oracle\": \"oracle\",\n \"snowflake\": \"snowflake\",\n \"bigquery\": \"bigquery\",\n \"redshift\": \"redshift\",\n \"presto\": \"presto\",\n \"trino\": \"trino\",\n \"druid\": \"druid\",\n \"hive\": \"hive\",\n \"spark\": \"spark\",\n \"databricks\": \"databricks\",\n }\n\n normalized = dialect_map.get(backend, backend)\n if normalized not in SUPPORTED_DIALECTS:\n log.explore(\"Unsupported dialect\", payload={\"backend\": backend, \"normalized\": normalized}, error=f\"Unsupported database dialect: {backend}\")\n raise ValueError(\n f\"Unsupported database dialect: '{backend}'. \"\n f\"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}\"\n )\n log.reflect(\"Dialect validated\", payload={\"dialect\": normalized})\n return normalized\n# #endregion get_dialect_from_database\n\n\n# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]\n# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.\n# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.\n# @POST Returns (columns_list, dialect_string) or raises on failure.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\ndef fetch_datasource_metadata(\n dataset_id: int,\n env_id: str,\n config_manager: ConfigManager,\n) -> Tuple[List[Dict[str, Any]], str]:\n \"\"\"Fetch column metadata and database dialect for a datasource from Superset.\"\"\"\n log.reason(\"Fetching datasource metadata\", payload={\n \"dataset_id\": dataset_id,\n \"env_id\": env_id,\n })\n # Find environment config\n environments = config_manager.get_environments()\n env_config = next((e for e in environments if e.id == env_id), None)\n if not env_config:\n log.explore(\"Environment not found\", payload={\"env_id\": env_id}, error=f\"Environment {env_id} not configured\")\n raise ValueError(f\"Superset environment '{env_id}' not found in configuration\")\n\n # Create Superset client and fetch dataset detail\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(dataset_id)\n\n # Extract columns\n raw_columns = dataset_detail.get(\"columns\", [])\n columns = []\n for col in raw_columns:\n col_name = col.get(\"name\") or col.get(\"column_name\")\n if not col_name:\n continue\n columns.append({\n \"name\": str(col_name),\n \"type\": col.get(\"type\"),\n \"is_physical\": col.get(\"is_physical\", True),\n \"is_dttm\": col.get(\"is_dttm\", False),\n \"description\": col.get(\"description\", \"\"),\n })\n\n # Extract database dialect from datasource\n database_info = dataset_detail.get(\"database\", {})\n if isinstance(database_info, dict):\n dialect = get_dialect_from_database(database_info)\n else:\n # Fallback: try to fetch database directly\n try:\n db_id = dataset_detail.get(\"database_id\")\n if db_id:\n db_record = client.get_database(int(db_id))\n db_result = db_record.get(\"result\", db_record) if isinstance(db_record, dict) else db_record\n dialect = get_dialect_from_database(db_result)\n else:\n raise ValueError(\"No database information available for this datasource\")\n except Exception as e:\n log.explore(\"Could not fetch database dialect\", error=str(e))\n raise ValueError(f\"Could not determine database dialect: {e}\")\n\n log.reflect(\"Metadata fetched\", payload={\"columns\": len(columns), \"dialect\": dialect})\n return columns, dialect\n# #endregion fetch_datasource_metadata\n\n\n# #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns]\n# @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries.\ndef detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]:\n \"\"\"Return names of columns that are virtual (not physical).\"\"\"\n return [col[\"name\"] for col in columns if not col.get(\"is_physical\", True)]\n# #endregion detect_virtual_columns\n\n\n# #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud]\n# @BRIEF Service for translation job CRUD with validation and Superset integration.\n# @PRE Database session and config manager are available.\n# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate).\n# @SIDE_EFFECT Queries Superset for dialect detection and column validation.\n# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob]\n# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits.\nclass TranslateJobService:\n\n def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None):\n self.db = db\n self.config_manager = config_manager\n self.current_user = current_user\n\n # #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]\n # @BRIEF List translation jobs with optional status filter and pagination.\n def list_jobs(\n self,\n page: int = 1,\n page_size: int = 20,\n status_filter: Optional[str] = None,\n ) -> Tuple[int, List[TranslationJob]]:\n query = self.db.query(TranslationJob)\n\n if status_filter:\n query = query.filter(TranslationJob.status == status_filter)\n\n total = query.count()\n offset = (page - 1) * page_size\n jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()\n\n return total, jobs\n # #endregion list_jobs\n\n # #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query]\n # @BRIEF Get a single translation job by ID, raising ValueError if not found.\n def get_job(self, job_id: str) -> TranslationJob:\n job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()\n if not job:\n raise ValueError(f\"Translation job '{job_id}' not found\")\n return job\n # #endregion get_job\n\n # #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create]\n # @BRIEF Create a new translation job with column validation and dialect detection.\n # @PRE payload contains valid job configuration (name, upsert_strategy, etc).\n # @POST Returns the created TranslationJob with database_dialect cached.\n # @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect.\n def create_job(self, payload: TranslateJobCreate) -> TranslationJob:\n log.reason(\"Creating translation job\", payload={\"name\": payload.name})\n\n # Validate: must have a translation column if datasource is configured\n if payload.source_datasource_id and not payload.translation_column:\n log.explore(\"Missing translation column\", error=\"translation_column required when source_datasource_id is set\", payload={\n \"source_datasource_id\": payload.source_datasource_id,\n })\n raise ValueError(\"A translation column is required when a datasource is selected\")\n\n # Validate upsert strategy\n valid_strategies = {\"MERGE\", \"INSERT\", \"UPDATE\"}\n if payload.upsert_strategy not in valid_strategies:\n raise ValueError(\n f\"Invalid upsert_strategy '{payload.upsert_strategy}'. \"\n f\"Must be one of: {', '.join(sorted(valid_strategies))}\"\n )\n\n # Detect database dialect and validate columns if datasource is specified\n dialect = payload.database_dialect\n if payload.source_datasource_id and (payload.environment_id or payload.source_dialect):\n # If no explicit dialect, try to detect it\n if not dialect:\n try:\n env_id = payload.environment_id or payload.source_dialect\n _, detected_dialect = fetch_datasource_metadata(\n int(payload.source_datasource_id),\n env_id,\n self.config_manager,\n )\n dialect = detected_dialect\n except Exception as e:\n log.explore(\"Dialect detection failed\", error=str(e))\n dialect = payload.source_dialect\n\n # Build job instance\n job = TranslationJob(\n id=str(uuid.uuid4()),\n name=payload.name,\n description=payload.description,\n source_dialect=payload.source_dialect,\n target_dialect=payload.target_dialect,\n database_dialect=dialect,\n source_datasource_id=payload.source_datasource_id,\n source_table=payload.source_table,\n target_schema=payload.target_schema,\n target_table=payload.target_table,\n source_key_cols=payload.source_key_cols or [],\n target_key_cols=payload.target_key_cols or [],\n translation_column=payload.translation_column,\n context_columns=payload.context_columns or [],\n target_language=payload.target_language,\n provider_id=payload.provider_id,\n batch_size=payload.batch_size,\n upsert_strategy=payload.upsert_strategy,\n environment_id=payload.environment_id,\n target_database_id=payload.target_database_id,\n status=\"DRAFT\",\n created_by=self.current_user,\n )\n self.db.add(job)\n self.db.flush()\n\n # Attach dictionaries\n if payload.dictionary_ids:\n for dict_id in payload.dictionary_ids:\n assoc = TranslationJobDictionary(\n id=str(uuid.uuid4()),\n job_id=job.id,\n dictionary_id=dict_id,\n )\n self.db.add(assoc)\n\n self.db.commit()\n self.db.refresh(job)\n log.reflect(\"Job created\", payload={\"job_id\": job.id})\n return job\n # #endregion create_job\n\n # #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update]\n # @BRIEF Update an existing translation job with optional dialect re-detection.\n # @PRE payload contains fields to update; job_id exists.\n # @POST Returns the updated TranslationJob with re-detected dialect if datasource changed.\n # @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.\n def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:\n log.reason(\"Updating translation job\", payload={\"job_id\": job_id})\n job = self.get_job(job_id)\n\n update_data = payload.model_dump(exclude_unset=True)\n dict_ids = update_data.pop(\"dictionary_ids\", None)\n\n for field, value in update_data.items():\n if hasattr(job, field):\n setattr(job, field, value)\n\n # Re-detect dialect if datasource changed\n if payload.source_datasource_id and not payload.database_dialect:\n try:\n env_id = (payload.environment_id or payload.source_dialect or job.environment_id or job.source_dialect)\n _, detected_dialect = fetch_datasource_metadata(\n int(payload.source_datasource_id),\n env_id,\n self.config_manager,\n )\n job.database_dialect = detected_dialect\n except Exception as e:\n log.explore(\"Dialect re-detection failed\", error=str(e))\n\n job.updated_at = datetime.now(timezone.utc)\n\n # Update dictionary associations if provided\n if dict_ids is not None:\n self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).delete()\n for dict_id in dict_ids:\n assoc = TranslationJobDictionary(\n id=str(uuid.uuid4()),\n job_id=job_id,\n dictionary_id=dict_id,\n )\n self.db.add(assoc)\n\n self.db.commit()\n self.db.refresh(job)\n log.reflect(\"Job updated\", payload={\"job_id\": job_id})\n return job\n # #endregion update_job\n\n # #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete]\n # @BRIEF Delete a translation job and its dictionary associations.\n # @PRE job_id must exist.\n # @POST Job and all related TranslationJobDictionary records are deleted.\n # @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows.\n def delete_job(self, job_id: str) -> None:\n log.reason(\"Deleting translation job\", payload={\"job_id\": job_id})\n job = self.get_job(job_id)\n\n # Delete dictionary associations\n self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).delete()\n\n self.db.delete(job)\n self.db.commit()\n log.reflect(\"Job deleted\", payload={\"job_id\": job_id})\n # #endregion delete_job\n\n # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate]\n # @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations.\n # @PRE job_id must exist.\n # @POST Returns the duplicated TranslationJob with status DRAFT.\n # @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows.\n def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob:\n log.reason(\"Duplicating translation job\", payload={\"job_id\": job_id, \"new_name\": new_name})\n source = self.get_job(job_id)\n\n # Copy all fields except id, created_at, updated_at, status\n new_job = TranslationJob(\n id=str(uuid.uuid4()),\n name=new_name or f\"{source.name} (Copy)\",\n description=source.description,\n source_dialect=source.source_dialect,\n target_dialect=source.target_dialect,\n database_dialect=source.database_dialect,\n source_datasource_id=source.source_datasource_id,\n source_table=source.source_table,\n target_schema=source.target_schema,\n target_table=source.target_table,\n source_key_cols=source.source_key_cols,\n target_key_cols=source.target_key_cols,\n translation_column=source.translation_column,\n context_columns=source.context_columns,\n target_language=source.target_language,\n provider_id=source.provider_id,\n batch_size=source.batch_size,\n upsert_strategy=source.upsert_strategy,\n environment_id=source.environment_id,\n target_database_id=source.target_database_id,\n status=\"DRAFT\",\n created_by=self.current_user,\n )\n self.db.add(new_job)\n self.db.flush()\n\n # Copy dictionary associations\n old_dicts = self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).all()\n for assoc in old_dicts:\n new_assoc = TranslationJobDictionary(\n id=str(uuid.uuid4()),\n job_id=new_job.id,\n dictionary_id=assoc.dictionary_id,\n )\n self.db.add(new_assoc)\n\n self.db.commit()\n self.db.refresh(new_job)\n log.reflect(\"Job duplicated\", payload={\"new_job_id\": new_job.id, \"source_job_id\": job_id})\n return new_job\n # #endregion duplicate_job\n\n # #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries]\n def get_job_dictionary_ids(self, job_id: str) -> List[str]:\n assocs = self.db.query(TranslationJobDictionary).filter(\n TranslationJobDictionary.job_id == job_id\n ).all()\n return [a.dictionary_id for a in assocs]\n # #endregion get_job_dictionary_ids\n\n # #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query]\n # @BRIEF List available Superset datasets for translation job creation with optional search filter.\n def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list:\n \"\"\"List Superset datasets available for translation.\"\"\"\n from ...core.superset_client import SupersetClient\n env = self.config_manager.get_environment(env_id)\n if not env:\n raise ValueError(f\"Environment '{env_id}' not found\")\n client = SupersetClient(env)\n _, datasets = client.get_datasets()\n result = []\n for ds in datasets:\n name = ds.get(\"table_name\", \"\")\n if search and search.lower() not in name.lower():\n continue\n db_info = ds.get(\"database\", {})\n backend = db_info.get(\"backend\", \"\")\n dialect = _extract_dialect(backend)\n result.append({\n \"id\": ds.get(\"id\"),\n \"table_name\": name,\n \"schema\": ds.get(\"schema\"),\n \"database_name\": db_info.get(\"database_name\", \"Unknown\"),\n \"database_dialect\": dialect,\n \"description\": ds.get(\"description\", \"\"),\n })\n return result\n # #endregion fetch_available_datasources\n# #endregion TranslateJobService\n\n\n# #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect]\ndef _extract_dialect(backend: str) -> str:\n \"\"\"Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...').\"\"\"\n if not backend:\n return \"unknown\"\n try:\n scheme = backend.split(\"://\")[0]\n dialect = scheme.split(\"+\")[0]\n return dialect.lower()\n except Exception:\n return \"unknown\"\n# #endregion _extract_dialect\n\n\n# #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization]\n# @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.\ndef job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse:\n return TranslateJobResponse(\n id=job.id,\n name=job.name,\n description=job.description,\n source_dialect=job.source_dialect,\n target_dialect=job.target_dialect,\n database_dialect=job.database_dialect,\n source_datasource_id=job.source_datasource_id,\n source_table=job.source_table,\n target_schema=job.target_schema,\n target_table=job.target_table,\n source_key_cols=job.source_key_cols or [],\n target_key_cols=job.target_key_cols or [],\n translation_column=job.translation_column,\n context_columns=job.context_columns or [],\n target_language=job.target_language,\n provider_id=job.provider_id,\n batch_size=job.batch_size or 50,\n upsert_strategy=job.upsert_strategy or \"MERGE\",\n status=job.status,\n created_by=job.created_by,\n created_at=job.created_at,\n updated_at=job.updated_at,\n dictionary_ids=dict_ids or [],\n environment_id=job.environment_id,\n target_database_id=job.target_database_id,\n )\n# #endregion job_to_response\n\n\n# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns]\n# @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse.\n# @PRE datasource_id is a valid Superset dataset ID.\n# @POST Returns DatasourceColumnsResponse with column metadata and database dialect.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\ndef get_datasource_columns(\n datasource_id: int,\n env_id: str,\n config_manager: ConfigManager,\n) -> DatasourceColumnsResponse:\n \"\"\"Fetch and return column metadata for a given Superset datasource.\"\"\"\n log.reason(\"Fetching datasource columns\", payload={\"datasource_id\": datasource_id, \"env_id\": env_id})\n\n # Find environment config\n environments = config_manager.get_environments()\n env_config = next((e for e in environments if e.id == env_id), None)\n if not env_config:\n log.explore(\"Environment not found\", payload={\"env_id\": env_id}, error=f\"Environment {env_id} not configured\")\n raise ValueError(f\"Superset environment '{env_id}' not found\")\n\n # Create Superset client\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(datasource_id)\n\n # Extract database dialect\n database_info = dataset_detail.get(\"database\", {})\n dialect = None\n if isinstance(database_info, dict):\n try:\n dialect = get_dialect_from_database(database_info)\n except (ValueError, KeyError):\n dialect = None\n if dialect is None:\n database_id = dataset_detail.get(\"database_id\")\n if database_id:\n db_record = client.get_database(int(database_id))\n db_result = db_record.get(\"result\", db_record) if isinstance(db_record, dict) else db_record\n dialect = get_dialect_from_database(db_result)\n else:\n log.explore(\"Could not determine dialect\", payload={\"datasource_id\": datasource_id}, error=f\"Could not determine dialect for datasource {datasource_id}\")\n raise ValueError(\"Could not determine database dialect for this datasource\")\n\n # Extract columns\n raw_columns = dataset_detail.get(\"columns\", [])\n columns = []\n for col in raw_columns:\n col_name = col.get(\"name\") or col.get(\"column_name\")\n if not col_name:\n continue\n columns.append(DatasourceColumnResponse(\n name=str(col_name),\n type=col.get(\"type\"),\n is_physical=col.get(\"is_physical\", True),\n is_dttm=col.get(\"is_dttm\", False),\n description=col.get(\"description\", \"\"),\n ))\n\n result = DatasourceColumnsResponse(\n datasource_id=datasource_id,\n datasource_name=dataset_detail.get(\"table_name\"),\n schema_name=dataset_detail.get(\"schema\"),\n database_dialect=dialect,\n columns=columns,\n )\n log.reflect(\"Datasource columns fetched\", payload={\n \"datasource_id\": datasource_id,\n \"columns_count\": len(columns),\n \"dialect\": dialect,\n })\n return result\n# #endregion get_datasource_columns\n\n# #endregion TranslateJobService\n" }, { "contract_id": "get_dialect_from_database", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 40, - "end_line": 92, + "start_line": 43, + "end_line": 95, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66211,14 +67220,14 @@ } ], "anchor_syntax": "region", - "body": "# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]\n# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.\n# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.\n# @POST Returns normalized dialect string or raises ValueError if unsupported.\n# @SIDE_EFFECT None — pure data extraction.\ndef get_dialect_from_database(database_record: Dict[str, Any]) -> str:\n \"\"\"Extract and validate dialect from Superset database record.\"\"\"\n logger.reason(\"Extracting dialect from database record\", {\n \"has_backend\": bool(database_record.get(\"backend\") or database_record.get(\"engine\")),\n })\n backend = (\n database_record.get(\"backend\")\n or database_record.get(\"engine\")\n or \"\"\n ).lower().strip()\n\n if not backend:\n logger.explore(\"Could not determine database dialect\", {\n \"record_keys\": list(database_record.keys()),\n })\n raise ValueError(\"Could not determine database dialect from connection\")\n\n # Map Superset backend names to normalized dialect\n dialect_map = {\n \"postgresql\": \"postgresql\",\n \"greenplum\": \"postgresql\",\n \"mysql\": \"mysql\",\n \"clickhouse\": \"clickhouse\",\n \"clickhousedb\": \"clickhouse\",\n \"sqlite\": \"sqlite\",\n \"mssql\": \"mssql\",\n \"oracle\": \"oracle\",\n \"snowflake\": \"snowflake\",\n \"bigquery\": \"bigquery\",\n \"redshift\": \"redshift\",\n \"presto\": \"presto\",\n \"trino\": \"trino\",\n \"druid\": \"druid\",\n \"hive\": \"hive\",\n \"spark\": \"spark\",\n \"databricks\": \"databricks\",\n }\n\n normalized = dialect_map.get(backend, backend)\n if normalized not in SUPPORTED_DIALECTS:\n logger.explore(\"Unsupported dialect\", {\"backend\": backend, \"normalized\": normalized})\n raise ValueError(\n f\"Unsupported database dialect: '{backend}'. \"\n f\"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}\"\n )\n logger.reflect(\"Dialect validated\", {\"dialect\": normalized})\n return normalized\n# #endregion get_dialect_from_database\n" + "body": "# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation]\n# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported.\n# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key.\n# @POST Returns normalized dialect string or raises ValueError if unsupported.\n# @SIDE_EFFECT None — pure data extraction.\ndef get_dialect_from_database(database_record: Dict[str, Any]) -> str:\n \"\"\"Extract and validate dialect from Superset database record.\"\"\"\n log.reason(\"Extracting dialect from database record\", payload={\n \"has_backend\": bool(database_record.get(\"backend\") or database_record.get(\"engine\")),\n })\n backend = (\n database_record.get(\"backend\")\n or database_record.get(\"engine\")\n or \"\"\n ).lower().strip()\n\n if not backend:\n log.explore(\"Could not determine database dialect\", error=\"No backend/engine field in database record\", payload={\n \"record_keys\": list(database_record.keys()),\n })\n raise ValueError(\"Could not determine database dialect from connection\")\n\n # Map Superset backend names to normalized dialect\n dialect_map = {\n \"postgresql\": \"postgresql\",\n \"greenplum\": \"postgresql\",\n \"mysql\": \"mysql\",\n \"clickhouse\": \"clickhouse\",\n \"clickhousedb\": \"clickhouse\",\n \"sqlite\": \"sqlite\",\n \"mssql\": \"mssql\",\n \"oracle\": \"oracle\",\n \"snowflake\": \"snowflake\",\n \"bigquery\": \"bigquery\",\n \"redshift\": \"redshift\",\n \"presto\": \"presto\",\n \"trino\": \"trino\",\n \"druid\": \"druid\",\n \"hive\": \"hive\",\n \"spark\": \"spark\",\n \"databricks\": \"databricks\",\n }\n\n normalized = dialect_map.get(backend, backend)\n if normalized not in SUPPORTED_DIALECTS:\n log.explore(\"Unsupported dialect\", payload={\"backend\": backend, \"normalized\": normalized}, error=f\"Unsupported database dialect: {backend}\")\n raise ValueError(\n f\"Unsupported database dialect: '{backend}'. \"\n f\"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}\"\n )\n log.reflect(\"Dialect validated\", payload={\"dialect\": normalized})\n return normalized\n# #endregion get_dialect_from_database\n" }, { "contract_id": "fetch_datasource_metadata", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 95, - "end_line": 156, + "start_line": 98, + "end_line": 159, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66256,14 +67265,14 @@ } ], "anchor_syntax": "region", - "body": "# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]\n# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.\n# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.\n# @POST Returns (columns_list, dialect_string) or raises on failure.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\ndef fetch_datasource_metadata(\n dataset_id: int,\n env_id: str,\n config_manager: ConfigManager,\n) -> Tuple[List[Dict[str, Any]], str]:\n \"\"\"Fetch column metadata and database dialect for a datasource from Superset.\"\"\"\n logger.reason(\"Fetching datasource metadata\", {\n \"dataset_id\": dataset_id,\n \"env_id\": env_id,\n })\n # Find environment config\n environments = config_manager.get_environments()\n env_config = next((e for e in environments if e.id == env_id), None)\n if not env_config:\n logger.explore(\"Environment not found\", {\"env_id\": env_id})\n raise ValueError(f\"Superset environment '{env_id}' not found in configuration\")\n\n # Create Superset client and fetch dataset detail\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(dataset_id)\n\n # Extract columns\n raw_columns = dataset_detail.get(\"columns\", [])\n columns = []\n for col in raw_columns:\n col_name = col.get(\"name\") or col.get(\"column_name\")\n if not col_name:\n continue\n columns.append({\n \"name\": str(col_name),\n \"type\": col.get(\"type\"),\n \"is_physical\": col.get(\"is_physical\", True),\n \"is_dttm\": col.get(\"is_dttm\", False),\n \"description\": col.get(\"description\", \"\"),\n })\n\n # Extract database dialect from datasource\n database_info = dataset_detail.get(\"database\", {})\n if isinstance(database_info, dict):\n dialect = get_dialect_from_database(database_info)\n else:\n # Fallback: try to fetch database directly\n try:\n db_id = dataset_detail.get(\"database_id\")\n if db_id:\n db_record = client.get_database(int(db_id))\n db_result = db_record.get(\"result\", db_record) if isinstance(db_record, dict) else db_record\n dialect = get_dialect_from_database(db_result)\n else:\n raise ValueError(\"No database information available for this datasource\")\n except Exception as e:\n logger.explore(\"Could not fetch database dialect\", {\"error\": str(e)})\n raise ValueError(f\"Could not determine database dialect: {e}\")\n\n logger.reflect(\"Metadata fetched\", {\"columns\": len(columns), \"dialect\": dialect})\n return columns, dialect\n# #endregion fetch_datasource_metadata\n" + "body": "# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata]\n# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset.\n# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials.\n# @POST Returns (columns_list, dialect_string) or raises on failure.\n# @SIDE_EFFECT Queries Superset API for dataset detail and database info.\ndef fetch_datasource_metadata(\n dataset_id: int,\n env_id: str,\n config_manager: ConfigManager,\n) -> Tuple[List[Dict[str, Any]], str]:\n \"\"\"Fetch column metadata and database dialect for a datasource from Superset.\"\"\"\n log.reason(\"Fetching datasource metadata\", payload={\n \"dataset_id\": dataset_id,\n \"env_id\": env_id,\n })\n # Find environment config\n environments = config_manager.get_environments()\n env_config = next((e for e in environments if e.id == env_id), None)\n if not env_config:\n log.explore(\"Environment not found\", payload={\"env_id\": env_id}, error=f\"Environment {env_id} not configured\")\n raise ValueError(f\"Superset environment '{env_id}' not found in configuration\")\n\n # Create Superset client and fetch dataset detail\n client = SupersetClient(env_config)\n dataset_detail = client.get_dataset_detail(dataset_id)\n\n # Extract columns\n raw_columns = dataset_detail.get(\"columns\", [])\n columns = []\n for col in raw_columns:\n col_name = col.get(\"name\") or col.get(\"column_name\")\n if not col_name:\n continue\n columns.append({\n \"name\": str(col_name),\n \"type\": col.get(\"type\"),\n \"is_physical\": col.get(\"is_physical\", True),\n \"is_dttm\": col.get(\"is_dttm\", False),\n \"description\": col.get(\"description\", \"\"),\n })\n\n # Extract database dialect from datasource\n database_info = dataset_detail.get(\"database\", {})\n if isinstance(database_info, dict):\n dialect = get_dialect_from_database(database_info)\n else:\n # Fallback: try to fetch database directly\n try:\n db_id = dataset_detail.get(\"database_id\")\n if db_id:\n db_record = client.get_database(int(db_id))\n db_result = db_record.get(\"result\", db_record) if isinstance(db_record, dict) else db_record\n dialect = get_dialect_from_database(db_result)\n else:\n raise ValueError(\"No database information available for this datasource\")\n except Exception as e:\n log.explore(\"Could not fetch database dialect\", error=str(e))\n raise ValueError(f\"Could not determine database dialect: {e}\")\n\n log.reflect(\"Metadata fetched\", payload={\"columns\": len(columns), \"dialect\": dialect})\n return columns, dialect\n# #endregion fetch_datasource_metadata\n" }, { "contract_id": "detect_virtual_columns", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 159, - "end_line": 164, + "start_line": 162, + "end_line": 167, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66295,8 +67304,8 @@ "contract_id": "get_job_dictionary_ids", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 418, - "end_line": 424, + "start_line": 421, + "end_line": 427, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -66321,8 +67330,8 @@ "contract_id": "fetch_available_datasources", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 426, - "end_line": 453, + "start_line": 429, + "end_line": 456, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66354,8 +67363,8 @@ "contract_id": "_extract_dialect", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 457, - "end_line": 468, + "start_line": 460, + "end_line": 471, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -66380,8 +67389,8 @@ "contract_id": "job_to_response", "contract_type": "Function", "file_path": "backend/src/plugins/translate/service.py", - "start_line": 471, - "end_line": 501, + "start_line": 474, + "end_line": 504, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66414,7 +67423,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/translate/sql_generator.py", "start_line": 1, - "end_line": 372, + "end_line": 376, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66465,14 +67474,14 @@ } ], "anchor_syntax": "region", - "body": "# #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect]\n# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.\n# @POST Returns safe SQL strings for the target dialect.\n\nfrom typing import Any, Dict, List, Optional, Tuple\nfrom datetime import datetime, timezone\nfrom ...core.logger import logger, belief_scope\n\n# PostgreSQL/Greenplum dialects that support ON CONFLICT\nPOSTGRESQL_DIALECTS = {\"postgresql\", \"redshift\", \"greenplum\"}\n# Dialects that use backtick or no quoting\nCLICKHOUSE_DIALECTS = {\"clickhouse\"}\n\n\n# #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp]\n# @BRIEF Detect Unix timestamp strings (seconds or millis) and convert to 'YYYY-MM-DD' for Date columns.\ndef _normalize_timestamp_value(value: Any) -> Optional[str]:\n \"\"\"Detect Unix timestamp values and convert to date string.\n\n Handles:\n - Integer/float Unix timestamps (seconds or milliseconds)\n - String representations of Unix timestamps (e.g. '1726358400000.0')\n\n Returns 'YYYY-MM-DD' if conversion succeeds, None if value is not a timestamp.\n \"\"\"\n # Try numeric conversion first\n try:\n ts = float(value)\n except (ValueError, TypeError):\n return None\n\n # Heuristic: Unix timestamps in seconds are ~10 digits (1e9 range for 2001-2033)\n # Unix timestamps in milliseconds are ~13 digits (1e12 range for 2001-2033)\n if 1e9 <= ts < 1e12:\n # Already in seconds\n pass\n elif 1e12 <= ts < 1e15:\n # Milliseconds — convert to seconds\n ts = ts / 1000.0\n else:\n # Not a plausible Unix timestamp\n return None\n\n try:\n dt = datetime.fromtimestamp(ts, tz=timezone.utc)\n return dt.strftime(\"%Y-%m-%d\")\n except (OSError, OverflowError, ValueError):\n return None\n# #endregion _normalize_timestamp_value\n\n\n# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting]\n# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse.\ndef _quote_identifier(identifier: str, dialect: str) -> str:\n \"\"\"Quote a SQL identifier per dialect rules.\"\"\"\n if not identifier:\n return identifier\n # Remove any existing quotes to avoid double-quoting\n cleaned = identifier.strip().strip('\"').strip('`').strip('[]')\n if dialect in POSTGRESQL_DIALECTS:\n return f'\"{cleaned}\"'\n elif dialect in CLICKHOUSE_DIALECTS:\n return f\"`{cleaned}`\"\n else:\n # Generic ANSI double-quote\n return f'\"{cleaned}\"'\n# #endregion _quote_identifier\n\n\n# #region _encode_sql_value [C:2] [TYPE Function] [SEMANTICS translate,sql,encoding]\n# @BRIEF Encode a Python value into a SQL-safe literal for INSERT VALUES, with ClickHouse timestamp normalization.\ndef _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str:\n \"\"\"Encode a Python value into a SQL-safe literal.\n\n For ClickHouse dialect, attempts to detect Unix timestamp strings\n and convert them to 'YYYY-MM-DD' format for Date column compatibility.\n \"\"\"\n if value is None:\n return \"NULL\"\n if isinstance(value, bool):\n return \"TRUE\" if value else \"FALSE\"\n\n # For ClickHouse: try to normalize timestamp-like string values\n if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value:\n normalized = _normalize_timestamp_value(value)\n if normalized:\n return f\"'{normalized}'\"\n\n if isinstance(value, (int, float)):\n return str(value)\n # String — escape single quotes by doubling them\n escaped = str(value).replace(\"'\", \"''\")\n return f\"'{escaped}'\"\n# #endregion _encode_sql_value\n\n\n# #region _build_values_clause [C:2] [TYPE Function] [SEMANTICS translate,sql,values]\n# @BRIEF Build a VALUES clause for multiple rows with per-column value encoding and dialect-aware quoting.\ndef _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect: Optional[str] = None) -> str:\n \"\"\"Build VALUES (...) clause for multiple rows.\n\n NOTE: columns may be quoted (e.g. '\"col\"' or '`col`') for the SQL column list,\n but row dicts have UNQUOTED keys. Strip quotes before value lookup.\n \"\"\"\n value_groups = []\n for row in rows:\n values = []\n for col in columns:\n # Strip quoting for value lookup — row keys are unquoted\n lookup_key = col.strip('\"').strip('`').strip('[]')\n val = row.get(lookup_key)\n values.append(_encode_sql_value(val, dialect=dialect))\n value_groups.append(f\"({', '.join(values)})\")\n return \",\\n\".join(value_groups)\n# #endregion _build_values_clause\n\n\n# #region generate_insert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,insert]\n# @BRIEF Generate a dialect-aware plain INSERT SQL statement for the given table, columns, and rows.\n# @RELATION DEPENDS_ON -> [_quote_identifier]\n# @RELATION DEPENDS_ON -> [_build_values_clause]\ndef generate_insert_sql(\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n dialect: Optional[str] = None,\n) -> str:\n \"\"\"Generate a plain INSERT SQL.\"\"\"\n with belief_scope(\"generate_insert_sql\"):\n if not target_table:\n raise ValueError(\"target_table is required for INSERT SQL generation\")\n if not columns:\n raise ValueError(\"At least one column is required for INSERT SQL generation\")\n if not rows:\n raise ValueError(\"At least one row is required for INSERT SQL generation\")\n\n col_list = \", \".join(columns)\n values = _build_values_clause(columns, rows, dialect=dialect)\n\n table_ref = target_table\n if target_schema:\n table_ref = f\"{target_schema}.{target_table}\"\n\n sql = f\"INSERT INTO {table_ref} ({col_list})\\nVALUES\\n{values};\"\n return sql\n# #endregion generate_insert_sql\n\n\n# #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert]\n# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING.\n# @RELATION DEPENDS_ON -> [_quote_identifier]\n# @RELATION DEPENDS_ON -> [_build_values_clause]\ndef generate_upsert_sql(\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n key_columns: List[str],\n rows: List[Dict[str, Any]],\n dialect: Optional[str] = None,\n) -> str:\n \"\"\"Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.\"\"\"\n with belief_scope(\"generate_upsert_sql\"):\n if not target_table:\n raise ValueError(\"target_table is required for UPSERT SQL generation\")\n if not columns:\n raise ValueError(\"At least one column is required for UPSERT SQL generation\")\n if not key_columns:\n raise ValueError(\"key_columns are required for UPSERT SQL generation\")\n if not rows:\n raise ValueError(\"At least one row is required for UPSERT SQL generation\")\n\n col_list = \", \".join(columns)\n key_list = \", \".join(key_columns)\n values = _build_values_clause(columns, rows, dialect=dialect)\n\n table_ref = target_table\n if target_schema:\n table_ref = f\"{target_schema}.{target_table}\"\n\n # Build SET clause: exclude key columns from update\n update_cols = [c for c in columns if c not in key_columns]\n if not update_cols:\n # If only key columns, use DO NOTHING\n conflict_action = \"DO NOTHING\"\n else:\n set_parts = [f\"{col} = EXCLUDED.{col}\" for col in update_cols]\n conflict_action = \"DO UPDATE SET\\n\" + \",\\n\".join(set_parts)\n\n sql = (\n f\"INSERT INTO {table_ref} ({col_list})\\n\"\n f\"VALUES\\n\"\n f\"{values}\\n\"\n f\"ON CONFLICT ({key_list}) {conflict_action};\"\n )\n return sql\n# #endregion generate_upsert_sql\n\n\n# #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator]\n# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support.\n# @PRE Job has target_schema, target_table, key columns configured.\n# @POST Returns generated SQL string for the target dialect.\n# @SIDE_EFFECT None — pure SQL generation.\n# @RELATION DEPENDS_ON -> [generate_insert_sql]\n# @RELATION DEPENDS_ON -> [generate_upsert_sql]\nclass SQLGenerator:\n\n # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]\n # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.\n # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.\n # @POST Returns tuple of (sql_string, statement_count).\n # @SIDE_EFFECT None — pure SQL generation.\n @staticmethod\n def generate(\n dialect: str,\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n key_columns: Optional[List[str]] = None,\n upsert_strategy: str = \"MERGE\",\n ) -> Tuple[str, int]:\n \"\"\"Generate dialect-appropriate INSERT/UPSERT SQL.\n\n Args:\n dialect: Target database dialect (e.g. 'postgresql', 'clickhouse').\n target_schema: Optional schema name.\n target_table: Target table name.\n columns: List of column names to insert.\n rows: List of row dicts with column values.\n key_columns: Key columns for conflict resolution (UPSERT).\n upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT).\n\n Returns:\n Tuple of (sql_string, row_count).\n \"\"\"\n with belief_scope(\"SQLGenerator.generate\"):\n logger.reason(\"Generating SQL\", {\n \"dialect\": dialect,\n \"schema\": target_schema,\n \"table\": target_table,\n \"columns\": len(columns),\n \"rows\": len(rows),\n \"strategy\": upsert_strategy,\n })\n\n # Validate inputs\n if not target_table:\n raise ValueError(\"target_table is required\")\n if not columns:\n raise ValueError(\"At least one column is required\")\n if not rows:\n raise ValueError(\"At least one row is required\")\n\n # Build fully qualified table reference\n table_ref = target_table\n if target_schema:\n quoted_schema = _quote_identifier(target_schema, dialect)\n quoted_table = _quote_identifier(target_table, dialect)\n table_ref = f\"{quoted_schema}.{quoted_table}\"\n else:\n table_ref = _quote_identifier(target_table, dialect)\n\n # Quote columns per dialect\n quoted_columns = [_quote_identifier(c, dialect) for c in columns]\n quoted_key_columns = (\n [_quote_identifier(k, dialect) for k in key_columns]\n if key_columns\n else []\n )\n\n # Generate SQL per dialect and strategy\n use_upsert = upsert_strategy.upper() == \"MERGE\" and key_columns\n\n if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS:\n # PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT\n if use_upsert:\n sql = generate_upsert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n key_columns=quoted_key_columns,\n rows=rows,\n dialect=dialect,\n )\n else:\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n elif dialect in CLICKHOUSE_DIALECTS:\n # ClickHouse: plain INSERT, no ON CONFLICT support\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n if use_upsert:\n logger.reason(\"ClickHouse UPSERT not supported; using plain INSERT\", {\n \"note\": \"ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.\",\n })\n else:\n # Fallback: plain INSERT\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n\n logger.reflect(\"SQL generated\", {\n \"dialect\": dialect,\n \"row_count\": len(rows),\n \"sql_length\": len(sql),\n })\n return sql, len(rows)\n # #endregion SQLGenerator.generate\n\n # #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch]\n # @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version).\n # @RELATION DEPENDS_ON -> [SQLGenerator.generate]\n @staticmethod\n def generate_batch(\n dialect: str,\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n key_columns: Optional[List[str]] = None,\n upsert_strategy: str = \"MERGE\",\n max_rows_per_statement: int = 500,\n ) -> List[Tuple[str, int]]:\n \"\"\"Generate SQL in batches, splitting large row sets into multiple statements.\n\n Returns:\n List of (sql_string, row_count) tuples.\n \"\"\"\n with belief_scope(\"SQLGenerator.generate_batch\"):\n if not rows:\n return []\n\n statements = []\n for i in range(0, len(rows), max_rows_per_statement):\n chunk = rows[i:i + max_rows_per_statement]\n sql, count = SQLGenerator.generate(\n dialect=dialect,\n target_schema=target_schema,\n target_table=target_table,\n columns=columns,\n rows=chunk,\n key_columns=key_columns,\n upsert_strategy=upsert_strategy,\n )\n statements.append((sql, count))\n\n return statements\n # #endregion SQLGenerator.generate_batch\n\n\n# #endregion SQLGenerator\n# #endregion SQLGenerator\n" + "body": "# #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect]\n# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding.\n# @LAYER Domain\n# @RELATION DEPENDS_ON -> [TranslationJob]\n# @RELATION DEPENDS_ON -> [TranslationRun]\n# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.\n# @POST Returns safe SQL strings for the target dialect.\n\nfrom typing import Any, Dict, List, Optional, Tuple\nfrom datetime import datetime, timezone\nfrom ...core.cot_logger import MarkerLogger\n\nfrom ...core.logger import belief_scope\n\nlog = MarkerLogger(\"SqlGenerator\")\n\n# PostgreSQL/Greenplum dialects that support ON CONFLICT\nPOSTGRESQL_DIALECTS = {\"postgresql\", \"redshift\", \"greenplum\"}\n# Dialects that use backtick or no quoting\nCLICKHOUSE_DIALECTS = {\"clickhouse\"}\n\n\n# #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp]\n# @BRIEF Detect Unix timestamp strings (seconds or millis) and convert to 'YYYY-MM-DD' for Date columns.\ndef _normalize_timestamp_value(value: Any) -> Optional[str]:\n \"\"\"Detect Unix timestamp values and convert to date string.\n\n Handles:\n - Integer/float Unix timestamps (seconds or milliseconds)\n - String representations of Unix timestamps (e.g. '1726358400000.0')\n\n Returns 'YYYY-MM-DD' if conversion succeeds, None if value is not a timestamp.\n \"\"\"\n # Try numeric conversion first\n try:\n ts = float(value)\n except (ValueError, TypeError):\n return None\n\n # Heuristic: Unix timestamps in seconds are ~10 digits (1e9 range for 2001-2033)\n # Unix timestamps in milliseconds are ~13 digits (1e12 range for 2001-2033)\n if 1e9 <= ts < 1e12:\n # Already in seconds\n pass\n elif 1e12 <= ts < 1e15:\n # Milliseconds — convert to seconds\n ts = ts / 1000.0\n else:\n # Not a plausible Unix timestamp\n return None\n\n try:\n dt = datetime.fromtimestamp(ts, tz=timezone.utc)\n return dt.strftime(\"%Y-%m-%d\")\n except (OSError, OverflowError, ValueError):\n return None\n# #endregion _normalize_timestamp_value\n\n\n# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting]\n# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse.\ndef _quote_identifier(identifier: str, dialect: str) -> str:\n \"\"\"Quote a SQL identifier per dialect rules.\"\"\"\n if not identifier:\n return identifier\n # Remove any existing quotes to avoid double-quoting\n cleaned = identifier.strip().strip('\"').strip('`').strip('[]')\n if dialect in POSTGRESQL_DIALECTS:\n return f'\"{cleaned}\"'\n elif dialect in CLICKHOUSE_DIALECTS:\n return f\"`{cleaned}`\"\n else:\n # Generic ANSI double-quote\n return f'\"{cleaned}\"'\n# #endregion _quote_identifier\n\n\n# #region _encode_sql_value [C:2] [TYPE Function] [SEMANTICS translate,sql,encoding]\n# @BRIEF Encode a Python value into a SQL-safe literal for INSERT VALUES, with ClickHouse timestamp normalization.\ndef _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str:\n \"\"\"Encode a Python value into a SQL-safe literal.\n\n For ClickHouse dialect, attempts to detect Unix timestamp strings\n and convert them to 'YYYY-MM-DD' format for Date column compatibility.\n \"\"\"\n if value is None:\n return \"NULL\"\n if isinstance(value, bool):\n return \"TRUE\" if value else \"FALSE\"\n\n # For ClickHouse: try to normalize timestamp-like string values\n if dialect in CLICKHOUSE_DIALECTS and isinstance(value, str) and value:\n normalized = _normalize_timestamp_value(value)\n if normalized:\n return f\"'{normalized}'\"\n\n if isinstance(value, (int, float)):\n return str(value)\n # String — escape single quotes by doubling them\n escaped = str(value).replace(\"'\", \"''\")\n return f\"'{escaped}'\"\n# #endregion _encode_sql_value\n\n\n# #region _build_values_clause [C:2] [TYPE Function] [SEMANTICS translate,sql,values]\n# @BRIEF Build a VALUES clause for multiple rows with per-column value encoding and dialect-aware quoting.\ndef _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect: Optional[str] = None) -> str:\n \"\"\"Build VALUES (...) clause for multiple rows.\n\n NOTE: columns may be quoted (e.g. '\"col\"' or '`col`') for the SQL column list,\n but row dicts have UNQUOTED keys. Strip quotes before value lookup.\n \"\"\"\n value_groups = []\n for row in rows:\n values = []\n for col in columns:\n # Strip quoting for value lookup — row keys are unquoted\n lookup_key = col.strip('\"').strip('`').strip('[]')\n val = row.get(lookup_key)\n values.append(_encode_sql_value(val, dialect=dialect))\n value_groups.append(f\"({', '.join(values)})\")\n return \",\\n\".join(value_groups)\n# #endregion _build_values_clause\n\n\n# #region generate_insert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,insert]\n# @BRIEF Generate a dialect-aware plain INSERT SQL statement for the given table, columns, and rows.\n# @RELATION DEPENDS_ON -> [_quote_identifier]\n# @RELATION DEPENDS_ON -> [_build_values_clause]\ndef generate_insert_sql(\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n dialect: Optional[str] = None,\n) -> str:\n \"\"\"Generate a plain INSERT SQL.\"\"\"\n with belief_scope(\"generate_insert_sql\"):\n if not target_table:\n raise ValueError(\"target_table is required for INSERT SQL generation\")\n if not columns:\n raise ValueError(\"At least one column is required for INSERT SQL generation\")\n if not rows:\n raise ValueError(\"At least one row is required for INSERT SQL generation\")\n\n col_list = \", \".join(columns)\n values = _build_values_clause(columns, rows, dialect=dialect)\n\n table_ref = target_table\n if target_schema:\n table_ref = f\"{target_schema}.{target_table}\"\n\n sql = f\"INSERT INTO {table_ref} ({col_list})\\nVALUES\\n{values};\"\n return sql\n# #endregion generate_insert_sql\n\n\n# #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert]\n# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING.\n# @RELATION DEPENDS_ON -> [_quote_identifier]\n# @RELATION DEPENDS_ON -> [_build_values_clause]\ndef generate_upsert_sql(\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n key_columns: List[str],\n rows: List[Dict[str, Any]],\n dialect: Optional[str] = None,\n) -> str:\n \"\"\"Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.\"\"\"\n with belief_scope(\"generate_upsert_sql\"):\n if not target_table:\n raise ValueError(\"target_table is required for UPSERT SQL generation\")\n if not columns:\n raise ValueError(\"At least one column is required for UPSERT SQL generation\")\n if not key_columns:\n raise ValueError(\"key_columns are required for UPSERT SQL generation\")\n if not rows:\n raise ValueError(\"At least one row is required for UPSERT SQL generation\")\n\n col_list = \", \".join(columns)\n key_list = \", \".join(key_columns)\n values = _build_values_clause(columns, rows, dialect=dialect)\n\n table_ref = target_table\n if target_schema:\n table_ref = f\"{target_schema}.{target_table}\"\n\n # Build SET clause: exclude key columns from update\n update_cols = [c for c in columns if c not in key_columns]\n if not update_cols:\n # If only key columns, use DO NOTHING\n conflict_action = \"DO NOTHING\"\n else:\n set_parts = [f\"{col} = EXCLUDED.{col}\" for col in update_cols]\n conflict_action = \"DO UPDATE SET\\n\" + \",\\n\".join(set_parts)\n\n sql = (\n f\"INSERT INTO {table_ref} ({col_list})\\n\"\n f\"VALUES\\n\"\n f\"{values}\\n\"\n f\"ON CONFLICT ({key_list}) {conflict_action};\"\n )\n return sql\n# #endregion generate_upsert_sql\n\n\n# #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator]\n# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support.\n# @PRE Job has target_schema, target_table, key columns configured.\n# @POST Returns generated SQL string for the target dialect.\n# @SIDE_EFFECT None — pure SQL generation.\n# @RELATION DEPENDS_ON -> [generate_insert_sql]\n# @RELATION DEPENDS_ON -> [generate_upsert_sql]\nclass SQLGenerator:\n\n # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]\n # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.\n # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.\n # @POST Returns tuple of (sql_string, statement_count).\n # @SIDE_EFFECT None — pure SQL generation.\n @staticmethod\n def generate(\n dialect: str,\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n key_columns: Optional[List[str]] = None,\n upsert_strategy: str = \"MERGE\",\n ) -> Tuple[str, int]:\n \"\"\"Generate dialect-appropriate INSERT/UPSERT SQL.\n\n Args:\n dialect: Target database dialect (e.g. 'postgresql', 'clickhouse').\n target_schema: Optional schema name.\n target_table: Target table name.\n columns: List of column names to insert.\n rows: List of row dicts with column values.\n key_columns: Key columns for conflict resolution (UPSERT).\n upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT).\n\n Returns:\n Tuple of (sql_string, row_count).\n \"\"\"\n with belief_scope(\"SQLGenerator.generate\"):\n log.reason(\"Generating SQL\", payload={\n \"dialect\": dialect,\n \"schema\": target_schema,\n \"table\": target_table,\n \"columns\": len(columns),\n \"rows\": len(rows),\n \"strategy\": upsert_strategy,\n })\n\n # Validate inputs\n if not target_table:\n raise ValueError(\"target_table is required\")\n if not columns:\n raise ValueError(\"At least one column is required\")\n if not rows:\n raise ValueError(\"At least one row is required\")\n\n # Build fully qualified table reference\n table_ref = target_table\n if target_schema:\n quoted_schema = _quote_identifier(target_schema, dialect)\n quoted_table = _quote_identifier(target_table, dialect)\n table_ref = f\"{quoted_schema}.{quoted_table}\"\n else:\n table_ref = _quote_identifier(target_table, dialect)\n\n # Quote columns per dialect\n quoted_columns = [_quote_identifier(c, dialect) for c in columns]\n quoted_key_columns = (\n [_quote_identifier(k, dialect) for k in key_columns]\n if key_columns\n else []\n )\n\n # Generate SQL per dialect and strategy\n use_upsert = upsert_strategy.upper() == \"MERGE\" and key_columns\n\n if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS:\n # PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT\n if use_upsert:\n sql = generate_upsert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n key_columns=quoted_key_columns,\n rows=rows,\n dialect=dialect,\n )\n else:\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n elif dialect in CLICKHOUSE_DIALECTS:\n # ClickHouse: plain INSERT, no ON CONFLICT support\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n if use_upsert:\n log.reason(\"ClickHouse UPSERT not supported; using plain INSERT\", payload={\n \"note\": \"ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.\",\n })\n else:\n # Fallback: plain INSERT\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n\n log.reflect(\"SQL generated\", payload={\n \"dialect\": dialect,\n \"row_count\": len(rows),\n \"sql_length\": len(sql),\n })\n return sql, len(rows)\n # #endregion SQLGenerator.generate\n\n # #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch]\n # @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version).\n # @RELATION DEPENDS_ON -> [SQLGenerator.generate]\n @staticmethod\n def generate_batch(\n dialect: str,\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n key_columns: Optional[List[str]] = None,\n upsert_strategy: str = \"MERGE\",\n max_rows_per_statement: int = 500,\n ) -> List[Tuple[str, int]]:\n \"\"\"Generate SQL in batches, splitting large row sets into multiple statements.\n\n Returns:\n List of (sql_string, row_count) tuples.\n \"\"\"\n with belief_scope(\"SQLGenerator.generate_batch\"):\n if not rows:\n return []\n\n statements = []\n for i in range(0, len(rows), max_rows_per_statement):\n chunk = rows[i:i + max_rows_per_statement]\n sql, count = SQLGenerator.generate(\n dialect=dialect,\n target_schema=target_schema,\n target_table=target_table,\n columns=columns,\n rows=chunk,\n key_columns=key_columns,\n upsert_strategy=upsert_strategy,\n )\n statements.append((sql, count))\n\n return statements\n # #endregion SQLGenerator.generate_batch\n\n\n# #endregion SQLGenerator\n# #endregion SQLGenerator\n" }, { "contract_id": "_normalize_timestamp_value", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 19, - "end_line": 53, + "start_line": 23, + "end_line": 57, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66504,8 +67513,8 @@ "contract_id": "_quote_identifier", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 56, - "end_line": 71, + "start_line": 60, + "end_line": 75, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66537,8 +67546,8 @@ "contract_id": "_encode_sql_value", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 74, - "end_line": 98, + "start_line": 78, + "end_line": 102, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66570,8 +67579,8 @@ "contract_id": "_build_values_clause", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 101, - "end_line": 119, + "start_line": 105, + "end_line": 123, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -66603,8 +67612,8 @@ "contract_id": "generate_insert_sql", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 122, - "end_line": 151, + "start_line": 126, + "end_line": 155, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -66649,8 +67658,8 @@ "contract_id": "generate_upsert_sql", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 154, - "end_line": 201, + "start_line": 158, + "end_line": 205, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -66695,8 +67704,8 @@ "contract_id": "SQLGenerator.generate", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 213, - "end_line": 328, + "start_line": 217, + "end_line": 332, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66734,14 +67743,14 @@ } ], "anchor_syntax": "region", - "body": " # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]\n # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.\n # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.\n # @POST Returns tuple of (sql_string, statement_count).\n # @SIDE_EFFECT None — pure SQL generation.\n @staticmethod\n def generate(\n dialect: str,\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n key_columns: Optional[List[str]] = None,\n upsert_strategy: str = \"MERGE\",\n ) -> Tuple[str, int]:\n \"\"\"Generate dialect-appropriate INSERT/UPSERT SQL.\n\n Args:\n dialect: Target database dialect (e.g. 'postgresql', 'clickhouse').\n target_schema: Optional schema name.\n target_table: Target table name.\n columns: List of column names to insert.\n rows: List of row dicts with column values.\n key_columns: Key columns for conflict resolution (UPSERT).\n upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT).\n\n Returns:\n Tuple of (sql_string, row_count).\n \"\"\"\n with belief_scope(\"SQLGenerator.generate\"):\n logger.reason(\"Generating SQL\", {\n \"dialect\": dialect,\n \"schema\": target_schema,\n \"table\": target_table,\n \"columns\": len(columns),\n \"rows\": len(rows),\n \"strategy\": upsert_strategy,\n })\n\n # Validate inputs\n if not target_table:\n raise ValueError(\"target_table is required\")\n if not columns:\n raise ValueError(\"At least one column is required\")\n if not rows:\n raise ValueError(\"At least one row is required\")\n\n # Build fully qualified table reference\n table_ref = target_table\n if target_schema:\n quoted_schema = _quote_identifier(target_schema, dialect)\n quoted_table = _quote_identifier(target_table, dialect)\n table_ref = f\"{quoted_schema}.{quoted_table}\"\n else:\n table_ref = _quote_identifier(target_table, dialect)\n\n # Quote columns per dialect\n quoted_columns = [_quote_identifier(c, dialect) for c in columns]\n quoted_key_columns = (\n [_quote_identifier(k, dialect) for k in key_columns]\n if key_columns\n else []\n )\n\n # Generate SQL per dialect and strategy\n use_upsert = upsert_strategy.upper() == \"MERGE\" and key_columns\n\n if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS:\n # PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT\n if use_upsert:\n sql = generate_upsert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n key_columns=quoted_key_columns,\n rows=rows,\n dialect=dialect,\n )\n else:\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n elif dialect in CLICKHOUSE_DIALECTS:\n # ClickHouse: plain INSERT, no ON CONFLICT support\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n if use_upsert:\n logger.reason(\"ClickHouse UPSERT not supported; using plain INSERT\", {\n \"note\": \"ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.\",\n })\n else:\n # Fallback: plain INSERT\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n\n logger.reflect(\"SQL generated\", {\n \"dialect\": dialect,\n \"row_count\": len(rows),\n \"sql_length\": len(sql),\n })\n return sql, len(rows)\n # #endregion SQLGenerator.generate\n" + "body": " # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate]\n # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration.\n # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty.\n # @POST Returns tuple of (sql_string, statement_count).\n # @SIDE_EFFECT None — pure SQL generation.\n @staticmethod\n def generate(\n dialect: str,\n target_schema: Optional[str],\n target_table: str,\n columns: List[str],\n rows: List[Dict[str, Any]],\n key_columns: Optional[List[str]] = None,\n upsert_strategy: str = \"MERGE\",\n ) -> Tuple[str, int]:\n \"\"\"Generate dialect-appropriate INSERT/UPSERT SQL.\n\n Args:\n dialect: Target database dialect (e.g. 'postgresql', 'clickhouse').\n target_schema: Optional schema name.\n target_table: Target table name.\n columns: List of column names to insert.\n rows: List of row dicts with column values.\n key_columns: Key columns for conflict resolution (UPSERT).\n upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT).\n\n Returns:\n Tuple of (sql_string, row_count).\n \"\"\"\n with belief_scope(\"SQLGenerator.generate\"):\n log.reason(\"Generating SQL\", payload={\n \"dialect\": dialect,\n \"schema\": target_schema,\n \"table\": target_table,\n \"columns\": len(columns),\n \"rows\": len(rows),\n \"strategy\": upsert_strategy,\n })\n\n # Validate inputs\n if not target_table:\n raise ValueError(\"target_table is required\")\n if not columns:\n raise ValueError(\"At least one column is required\")\n if not rows:\n raise ValueError(\"At least one row is required\")\n\n # Build fully qualified table reference\n table_ref = target_table\n if target_schema:\n quoted_schema = _quote_identifier(target_schema, dialect)\n quoted_table = _quote_identifier(target_table, dialect)\n table_ref = f\"{quoted_schema}.{quoted_table}\"\n else:\n table_ref = _quote_identifier(target_table, dialect)\n\n # Quote columns per dialect\n quoted_columns = [_quote_identifier(c, dialect) for c in columns]\n quoted_key_columns = (\n [_quote_identifier(k, dialect) for k in key_columns]\n if key_columns\n else []\n )\n\n # Generate SQL per dialect and strategy\n use_upsert = upsert_strategy.upper() == \"MERGE\" and key_columns\n\n if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS:\n # PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT\n if use_upsert:\n sql = generate_upsert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n key_columns=quoted_key_columns,\n rows=rows,\n dialect=dialect,\n )\n else:\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n elif dialect in CLICKHOUSE_DIALECTS:\n # ClickHouse: plain INSERT, no ON CONFLICT support\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n if use_upsert:\n log.reason(\"ClickHouse UPSERT not supported; using plain INSERT\", payload={\n \"note\": \"ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.\",\n })\n else:\n # Fallback: plain INSERT\n sql = generate_insert_sql(\n target_schema=None,\n target_table=table_ref,\n columns=quoted_columns,\n rows=rows,\n dialect=dialect,\n )\n\n log.reflect(\"SQL generated\", payload={\n \"dialect\": dialect,\n \"row_count\": len(rows),\n \"sql_length\": len(sql),\n })\n return sql, len(rows)\n # #endregion SQLGenerator.generate\n" }, { "contract_id": "SQLGenerator.generate_batch", "contract_type": "Function", "file_path": "backend/src/plugins/translate/sql_generator.py", - "start_line": 330, - "end_line": 368, + "start_line": 334, + "end_line": 372, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -66781,7 +67790,7 @@ "contract_type": "Module", "file_path": "backend/src/plugins/translate/superset_executor.py", "start_line": 1, - "end_line": 358, + "end_line": 363, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -66828,14 +67837,14 @@ } ], "anchor_syntax": "region", - "body": "# #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute]\n# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry.\n# @LAYER Infra\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @PRE Valid Superset environment configuration and authenticated client.\n# @POST SQL is submitted to Superset SQL Lab; execution reference is returned.\n# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.\n# @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message]\n# @INVARIANT Polling respects max_polls limit; timeout returns structured error.\n# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.\n# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.\n\nfrom typing import Any, Dict, List, Optional, Tuple, Union\nfrom datetime import datetime, timezone\nimport json\nimport time\nimport uuid\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.config_manager import ConfigManager\nfrom ...core.superset_client import SupersetClient\n\n\n# #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab]\n# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.\n# @PRE Valid environment ID and ConfigManager.\n# @POST SQL submitted to Superset; execution reference recorded; polling completes.\n# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.\n# @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status]\n# @INVARIANT database_id resolved lazily; polling capped at max_polls.\nclass SupersetSqlLabExecutor:\n\n def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None):\n self.config_manager = config_manager\n self.env_id = env_id\n self._client: Optional[SupersetClient] = None\n self._database_id: Optional[int] = database_id\n\n # #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client]\n # @BRIEF Lazy-initialize SupersetClient for the configured environment.\n # @PRE env_id must correspond to a valid environment config.\n # @POST Returns authenticated SupersetClient.\n # @SIDE_EFFECT Authenticates against Superset API on first call.\n def _get_client(self) -> SupersetClient:\n with belief_scope(\"SupersetSqlLabExecutor._get_client\"):\n if self._client is not None:\n return self._client\n\n environments = self.config_manager.get_environments()\n env_config = next((e for e in environments if e.id == self.env_id), None)\n if not env_config:\n raise ValueError(f\"Superset environment '{self.env_id}' not found\")\n\n self._client = SupersetClient(env_config)\n return self._client\n # #endregion _get_client\n\n # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]\n # @BRIEF Resolve the target database ID from the environment by name or default.\n # @PRE Superset environment is reachable and has databases.\n # @POST Returns database_id integer or raises ValueError.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_id(self, database_name: Optional[str] = None) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_id\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n if database_name:\n for db in databases:\n if db.get(\"database_name\", \"\").lower() == database_name.lower():\n self._database_id = db[\"id\"]\n logger.reason(\"Resolved database ID by name\", {\n \"database_name\": database_name,\n \"database_id\": self._database_id,\n })\n return self._database_id\n raise ValueError(\n f\"Database '{database_name}' not found in Superset environment\"\n )\n\n # Default: use first database\n self._database_id = databases[0][\"id\"]\n logger.reason(\"Using default database\", {\n \"database_name\": databases[0].get(\"database_name\"),\n \"database_id\": self._database_id,\n })\n return self._database_id\n # #endregion resolve_database_id\n\n # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]\n # @BRIEF Resolve a database UUID to a numeric database ID from Superset.\n # @PRE uuid_str is a valid Superset database UUID.\n # @POST Returns numeric database_id or raises ValueError if not found.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_uuid(self, uuid_str: str) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_uuid\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"uuid\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n for db in databases:\n db_uuid = db.get(\"uuid\") or \"\"\n if db_uuid.lower() == uuid_str.lower():\n db_id = db[\"id\"]\n self._database_id = db_id\n logger.reason(\"Resolved database ID by UUID\", {\n \"uuid\": uuid_str,\n \"database_id\": db_id,\n \"database_name\": db.get(\"database_name\"),\n })\n return db_id\n\n raise ValueError(\n f\"Database with UUID '{uuid_str}' not found in Superset environment\"\n )\n # #endregion resolve_database_uuid\n\n # #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]\n # @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.\n # @PRE sql is valid SQL string. database_id is a valid Superset DB ID.\n # @POST Returns execution result dict with query_id and status.\n # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.\n def execute_sql(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_sql\"):\n client = self._get_client()\n db_id = database_id or self._database_id\n if db_id is None:\n db_id = self.resolve_database_id()\n # Handle UUID string passed as database_id — resolve to numeric ID\n if isinstance(db_id, str):\n try:\n db_id = int(db_id)\n except (ValueError, TypeError):\n db_id = self.resolve_database_uuid(db_id)\n\n logger.reason(\"Submitting SQL to Superset SQL Lab\", {\n \"database_id\": db_id,\n \"sql_length\": len(sql),\n })\n\n payload = {\n \"database_id\": db_id,\n \"sql\": sql,\n \"client_id\": uuid.uuid4().hex[:10],\n \"schema\": None,\n }\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/sqllab/execute/\",\n data=json.dumps(payload),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n logger.explore(\"SQL Lab execute failed\", {\"error\": str(e)})\n raise ValueError(f\"Superset SQL Lab execute failed: {e}\")\n\n # Parse response — try multiple formats\n result = response if isinstance(response, dict) else {}\n query_id = (result.get(\"query_id\")\n or result.get(\"id\")\n or result.get(\"sql_lab_session_ref\")\n or (result.get(\"result\") or {}).get(\"query_id\")\n or (result.get(\"result\") or {}).get(\"id\"))\n status = result.get(\"status\", \"unknown\")\n\n logger.reason(\"SQL Lab execute response\", {\n \"query_id\": query_id,\n \"status\": status,\n \"has_query_id\": query_id is not None,\n \"has_errors\": \"errors\" in result or \"error\" in result,\n \"response_keys\": list(result.keys()) if isinstance(result, dict) else type(result).__name__,\n \"response_preview\": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],\n })\n\n return {\n \"query_id\": query_id,\n \"status\": status,\n \"raw_response\": result,\n \"database_id\": db_id,\n }\n # #endregion execute_sql\n\n # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]\n # @BRIEF Poll Superset for SQL execution status until completion or timeout.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns final execution status dict with success/failure/timeout.\n # @SIDE_EFFECT Makes HTTP GET requests to Superset API.\n def poll_execution_status(\n self,\n query_id: str,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.poll_execution_status\"):\n client = self._get_client()\n\n logger.reason(\"Polling SQL execution status\", {\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n \"interval\": poll_interval_seconds,\n })\n\n for attempt in range(max_polls):\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}\",\n )\n result = response if isinstance(response, dict) else {}\n status = result.get(\"status\", \"unknown\")\n state = result.get(\"state\", result.get(\"status\", \"\"))\n\n # Terminal states\n if state in (\"success\", \"finished\", \"completed\"):\n logger.reason(\"SQL execution completed\", {\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"rows_affected\": result.get(\"rows\"),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"success\",\n \"state\": state,\n \"rows_affected\": result.get(\"rows\"),\n \"error_message\": result.get(\"error_message\"),\n \"results\": result.get(\"results\"),\n \"completed_on\": result.get(\"completed_on\"),\n }\n\n if state in (\"failed\", \"error\", \"stopped\"):\n error_msg = result.get(\"error_message\", \"Unknown error\")\n logger.explore(\"SQL execution failed\", {\n \"query_id\": query_id,\n \"state\": state,\n \"error\": error_msg,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"state\": state,\n \"error_message\": error_msg,\n \"results\": None,\n }\n\n if state in (\"pending\", \"running\", \"started\"):\n time.sleep(poll_interval_seconds)\n continue\n\n # Unknown state — treat as still running\n time.sleep(poll_interval_seconds)\n\n except Exception as e:\n logger.explore(\"Polling error, retrying\", {\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"error\": str(e),\n })\n time.sleep(poll_interval_seconds)\n continue\n\n # Timeout\n logger.explore(\"SQL execution polling timed out\", {\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"timeout\",\n \"error_message\": f\"Polling timed out after {max_polls} attempts\",\n \"results\": None,\n }\n # #endregion poll_execution_status\n\n # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]\n # @BRIEF Execute SQL and wait for completion. One-shot convenience method.\n # @PRE sql is valid SQL.\n # @POST Returns final execution result.\n # @SIDE_EFFECT Makes HTTP calls to Superset API.\n def execute_and_poll(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_and_poll\"):\n exec_result = self.execute_sql(\n sql=sql,\n database_id=database_id,\n )\n\n query_id = exec_result.get(\"query_id\")\n if not query_id:\n logger.explore(\"No query_id from SQL Lab execute\", {\n \"raw_response\": exec_result.get(\"raw_response\"),\n })\n return {\n \"status\": \"failed\",\n \"error_message\": \"No query_id returned from SQL Lab\",\n \"query_id\": None,\n }\n\n return self.poll_execution_status(\n query_id=str(query_id),\n max_polls=max_polls,\n poll_interval_seconds=poll_interval_seconds,\n )\n # #endregion execute_and_poll\n\n # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]\n # @BRIEF Fetch the results of a completed query from Superset.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns query results if available, or error dict.\n # @SIDE_EFFECT Makes HTTP GET to Superset API.\n def get_query_results(self, query_id: str) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.get_query_results\"):\n client = self._get_client()\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}/results\",\n )\n result = response if isinstance(response, dict) else {}\n return {\n \"query_id\": query_id,\n \"status\": \"success\" if result.get(\"results\") else \"no_results\",\n \"results\": result.get(\"results\"),\n }\n except Exception as e:\n logger.explore(\"Failed to fetch query results\", {\n \"query_id\": query_id,\n \"error\": str(e),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"error_message\": str(e),\n \"results\": None,\n }\n # #endregion get_query_results\n\n\n# #endregion SupersetSqlLabExecutor\n# #endregion SupersetSqlLabExecutor\n" + "body": "# #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute]\n# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry.\n# @LAYER Infra\n# @RELATION DEPENDS_ON -> [SupersetClient]\n# @RELATION DEPENDS_ON -> [ConfigManager]\n# @PRE Valid Superset environment configuration and authenticated client.\n# @POST SQL is submitted to Superset SQL Lab; execution reference is returned.\n# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.\n# @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message]\n# @INVARIANT Polling respects max_polls limit; timeout returns structured error.\n# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.\n# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.\n\nfrom typing import Any, Dict, List, Optional, Tuple, Union\nfrom datetime import datetime, timezone\nimport json\nimport time\nimport uuid\n\nfrom ...core.logger import logger, belief_scope\nfrom ...core.cot_logger import MarkerLogger\nfrom ...core.config_manager import ConfigManager\nfrom ...core.superset_client import SupersetClient\n\nlog = MarkerLogger(\"SupersetExecutor\")\n\n# #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab]\n# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.\n# @PRE Valid environment ID and ConfigManager.\n# @POST SQL submitted to Superset; execution reference recorded; polling completes.\n# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.\n# @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status]\n# @INVARIANT database_id resolved lazily; polling capped at max_polls.\nclass SupersetSqlLabExecutor:\n\n def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None):\n self.config_manager = config_manager\n self.env_id = env_id\n self._client: Optional[SupersetClient] = None\n self._database_id: Optional[int] = database_id\n\n # #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client]\n # @BRIEF Lazy-initialize SupersetClient for the configured environment.\n # @PRE env_id must correspond to a valid environment config.\n # @POST Returns authenticated SupersetClient.\n # @SIDE_EFFECT Authenticates against Superset API on first call.\n def _get_client(self) -> SupersetClient:\n with belief_scope(\"SupersetSqlLabExecutor._get_client\"):\n if self._client is not None:\n return self._client\n\n environments = self.config_manager.get_environments()\n env_config = next((e for e in environments if e.id == self.env_id), None)\n if not env_config:\n raise ValueError(f\"Superset environment '{self.env_id}' not found\")\n\n self._client = SupersetClient(env_config)\n return self._client\n # #endregion _get_client\n\n # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]\n # @BRIEF Resolve the target database ID from the environment by name or default.\n # @PRE Superset environment is reachable and has databases.\n # @POST Returns database_id integer or raises ValueError.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_id(self, database_name: Optional[str] = None) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_id\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n if database_name:\n for db in databases:\n if db.get(\"database_name\", \"\").lower() == database_name.lower():\n self._database_id = db[\"id\"]\n log.reason(\"Resolved database ID by name\", payload={\n \"database_name\": database_name,\n \"database_id\": self._database_id,\n })\n return self._database_id\n raise ValueError(\n f\"Database '{database_name}' not found in Superset environment\"\n )\n\n # Default: use first database\n self._database_id = databases[0][\"id\"]\n log.reason(\"Using default database\", payload={\n \"database_name\": databases[0].get(\"database_name\"),\n \"database_id\": self._database_id,\n })\n return self._database_id\n # #endregion resolve_database_id\n\n # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]\n # @BRIEF Resolve a database UUID to a numeric database ID from Superset.\n # @PRE uuid_str is a valid Superset database UUID.\n # @POST Returns numeric database_id or raises ValueError if not found.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_uuid(self, uuid_str: str) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_uuid\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"uuid\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n for db in databases:\n db_uuid = db.get(\"uuid\") or \"\"\n if db_uuid.lower() == uuid_str.lower():\n db_id = db[\"id\"]\n self._database_id = db_id\n log.reason(\"Resolved database ID by UUID\", payload={\n \"uuid\": uuid_str,\n \"database_id\": db_id,\n \"database_name\": db.get(\"database_name\"),\n })\n return db_id\n\n raise ValueError(\n f\"Database with UUID '{uuid_str}' not found in Superset environment\"\n )\n # #endregion resolve_database_uuid\n\n # #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]\n # @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.\n # @PRE sql is valid SQL string. database_id is a valid Superset DB ID.\n # @POST Returns execution result dict with query_id and status.\n # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.\n def execute_sql(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_sql\"):\n client = self._get_client()\n db_id = database_id or self._database_id\n if db_id is None:\n db_id = self.resolve_database_id()\n # Handle UUID string passed as database_id — resolve to numeric ID\n if isinstance(db_id, str):\n try:\n db_id = int(db_id)\n except (ValueError, TypeError):\n db_id = self.resolve_database_uuid(db_id)\n\n log.reason(\"Submitting SQL to Superset SQL Lab\", payload={\n \"database_id\": db_id,\n \"sql_length\": len(sql),\n })\n\n payload = {\n \"database_id\": db_id,\n \"sql\": sql,\n \"client_id\": uuid.uuid4().hex[:10],\n \"schema\": None,\n }\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/sqllab/execute/\",\n data=json.dumps(payload),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n log.explore(\"SQL Lab execute failed\", error=str(e))\n raise ValueError(f\"Superset SQL Lab execute failed: {e}\")\n\n # Parse response — try multiple formats\n result = response if isinstance(response, dict) else {}\n query_id = (result.get(\"query_id\")\n or result.get(\"id\")\n or result.get(\"sql_lab_session_ref\")\n or (result.get(\"result\") or {}).get(\"query_id\")\n or (result.get(\"result\") or {}).get(\"id\"))\n status = result.get(\"status\", \"unknown\")\n\n log.reason(\"SQL Lab execute response\", payload={\n \"query_id\": query_id,\n \"status\": status,\n \"has_query_id\": query_id is not None,\n \"has_errors\": \"errors\" in result or \"error\" in result,\n \"response_keys\": list(result.keys()) if isinstance(result, dict) else type(result).__name__,\n \"response_preview\": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],\n })\n\n return {\n \"query_id\": query_id,\n \"status\": status,\n \"raw_response\": result,\n \"database_id\": db_id,\n }\n # #endregion execute_sql\n\n # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]\n # @BRIEF Poll Superset for SQL execution status until completion or timeout.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns final execution status dict with success/failure/timeout.\n # @SIDE_EFFECT Makes HTTP GET requests to Superset API.\n def poll_execution_status(\n self,\n query_id: str,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.poll_execution_status\"):\n client = self._get_client()\n\n log.reason(\"Polling SQL execution status\", payload={\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n \"interval\": poll_interval_seconds,\n })\n\n for attempt in range(max_polls):\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}\",\n )\n result = response if isinstance(response, dict) else {}\n status = result.get(\"status\", \"unknown\")\n state = result.get(\"state\", result.get(\"status\", \"\"))\n\n # Terminal states\n if state in (\"success\", \"finished\", \"completed\"):\n log.reason(\"SQL execution completed\", payload={\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"rows_affected\": result.get(\"rows\"),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"success\",\n \"state\": state,\n \"rows_affected\": result.get(\"rows\"),\n \"error_message\": result.get(\"error_message\"),\n \"results\": result.get(\"results\"),\n \"completed_on\": result.get(\"completed_on\"),\n }\n\n if state in (\"failed\", \"error\", \"stopped\"):\n error_msg = result.get(\"error_message\", \"Unknown error\")\n log.explore(\"SQL execution failed\", error=\"SQL query execution returned failed/error state\",\n payload={\n \"query_id\": query_id,\n \"state\": state,\n \"error\": error_msg,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"state\": state,\n \"error_message\": error_msg,\n \"results\": None,\n }\n\n if state in (\"pending\", \"running\", \"started\"):\n time.sleep(poll_interval_seconds)\n continue\n\n # Unknown state — treat as still running\n time.sleep(poll_interval_seconds)\n\n except Exception as e:\n log.explore(\"Polling error, retrying\", error=\"Polling encountered error, will retry\",\n payload={\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"error\": str(e),\n })\n time.sleep(poll_interval_seconds)\n continue\n\n # Timeout\n log.explore(\"SQL execution polling timed out\", error=\"Polling timed out\", payload={\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"timeout\",\n \"error_message\": f\"Polling timed out after {max_polls} attempts\",\n \"results\": None,\n }\n # #endregion poll_execution_status\n\n # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]\n # @BRIEF Execute SQL and wait for completion. One-shot convenience method.\n # @PRE sql is valid SQL.\n # @POST Returns final execution result.\n # @SIDE_EFFECT Makes HTTP calls to Superset API.\n def execute_and_poll(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_and_poll\"):\n exec_result = self.execute_sql(\n sql=sql,\n database_id=database_id,\n )\n\n query_id = exec_result.get(\"query_id\")\n if not query_id:\n log.explore(\"No query_id from SQL Lab execute\", error=\"No query_id returned\", payload={\n \"raw_response\": exec_result.get(\"raw_response\"),\n })\n return {\n \"status\": \"failed\",\n \"error_message\": \"No query_id returned from SQL Lab\",\n \"query_id\": None,\n }\n\n return self.poll_execution_status(\n query_id=str(query_id),\n max_polls=max_polls,\n poll_interval_seconds=poll_interval_seconds,\n )\n # #endregion execute_and_poll\n\n # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]\n # @BRIEF Fetch the results of a completed query from Superset.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns query results if available, or error dict.\n # @SIDE_EFFECT Makes HTTP GET to Superset API.\n def get_query_results(self, query_id: str) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.get_query_results\"):\n client = self._get_client()\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}/results\",\n )\n result = response if isinstance(response, dict) else {}\n return {\n \"query_id\": query_id,\n \"status\": \"success\" if result.get(\"results\") else \"no_results\",\n \"results\": result.get(\"results\"),\n }\n except Exception as e:\n log.explore(\"Failed to fetch query results\", error=\"Failed to fetch query results from Superset\",\n payload={\n \"query_id\": query_id,\n \"error\": str(e),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"error_message\": str(e),\n \"results\": None,\n }\n # #endregion get_query_results\n\n\n# #endregion SupersetSqlLabExecutor\n# #endregion SupersetSqlLabExecutor\n" }, { "contract_id": "resolve_database_id", "contract_type": "Function", "file_path": "backend/src/plugins/translate/superset_executor.py", - "start_line": 59, - "end_line": 93, + "start_line": 61, + "end_line": 95, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66873,14 +67882,14 @@ } ], "anchor_syntax": "region", - "body": " # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]\n # @BRIEF Resolve the target database ID from the environment by name or default.\n # @PRE Superset environment is reachable and has databases.\n # @POST Returns database_id integer or raises ValueError.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_id(self, database_name: Optional[str] = None) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_id\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n if database_name:\n for db in databases:\n if db.get(\"database_name\", \"\").lower() == database_name.lower():\n self._database_id = db[\"id\"]\n logger.reason(\"Resolved database ID by name\", {\n \"database_name\": database_name,\n \"database_id\": self._database_id,\n })\n return self._database_id\n raise ValueError(\n f\"Database '{database_name}' not found in Superset environment\"\n )\n\n # Default: use first database\n self._database_id = databases[0][\"id\"]\n logger.reason(\"Using default database\", {\n \"database_name\": databases[0].get(\"database_name\"),\n \"database_id\": self._database_id,\n })\n return self._database_id\n # #endregion resolve_database_id\n" + "body": " # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve]\n # @BRIEF Resolve the target database ID from the environment by name or default.\n # @PRE Superset environment is reachable and has databases.\n # @POST Returns database_id integer or raises ValueError.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_id(self, database_name: Optional[str] = None) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_id\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n if database_name:\n for db in databases:\n if db.get(\"database_name\", \"\").lower() == database_name.lower():\n self._database_id = db[\"id\"]\n log.reason(\"Resolved database ID by name\", payload={\n \"database_name\": database_name,\n \"database_id\": self._database_id,\n })\n return self._database_id\n raise ValueError(\n f\"Database '{database_name}' not found in Superset environment\"\n )\n\n # Default: use first database\n self._database_id = databases[0][\"id\"]\n log.reason(\"Using default database\", payload={\n \"database_name\": databases[0].get(\"database_name\"),\n \"database_id\": self._database_id,\n })\n return self._database_id\n # #endregion resolve_database_id\n" }, { "contract_id": "resolve_database_uuid", "contract_type": "Function", "file_path": "backend/src/plugins/translate/superset_executor.py", - "start_line": 95, - "end_line": 124, + "start_line": 97, + "end_line": 126, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66918,14 +67927,14 @@ } ], "anchor_syntax": "region", - "body": " # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]\n # @BRIEF Resolve a database UUID to a numeric database ID from Superset.\n # @PRE uuid_str is a valid Superset database UUID.\n # @POST Returns numeric database_id or raises ValueError if not found.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_uuid(self, uuid_str: str) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_uuid\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"uuid\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n for db in databases:\n db_uuid = db.get(\"uuid\") or \"\"\n if db_uuid.lower() == uuid_str.lower():\n db_id = db[\"id\"]\n self._database_id = db_id\n logger.reason(\"Resolved database ID by UUID\", {\n \"uuid\": uuid_str,\n \"database_id\": db_id,\n \"database_name\": db.get(\"database_name\"),\n })\n return db_id\n\n raise ValueError(\n f\"Database with UUID '{uuid_str}' not found in Superset environment\"\n )\n # #endregion resolve_database_uuid\n" + "body": " # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid]\n # @BRIEF Resolve a database UUID to a numeric database ID from Superset.\n # @PRE uuid_str is a valid Superset database UUID.\n # @POST Returns numeric database_id or raises ValueError if not found.\n # @SIDE_EFFECT Fetches databases list from Superset API.\n def resolve_database_uuid(self, uuid_str: str) -> int:\n with belief_scope(\"SupersetSqlLabExecutor.resolve_database_uuid\"):\n client = self._get_client()\n _, databases = client.get_databases(\n query={\"columns\": [\"id\", \"uuid\", \"database_name\"]}\n )\n if not databases:\n raise ValueError(\"No databases found in Superset environment\")\n\n for db in databases:\n db_uuid = db.get(\"uuid\") or \"\"\n if db_uuid.lower() == uuid_str.lower():\n db_id = db[\"id\"]\n self._database_id = db_id\n log.reason(\"Resolved database ID by UUID\", payload={\n \"uuid\": uuid_str,\n \"database_id\": db_id,\n \"database_name\": db.get(\"database_name\"),\n })\n return db_id\n\n raise ValueError(\n f\"Database with UUID '{uuid_str}' not found in Superset environment\"\n )\n # #endregion resolve_database_uuid\n" }, { "contract_id": "execute_sql", "contract_type": "Function", "file_path": "backend/src/plugins/translate/superset_executor.py", - "start_line": 126, - "end_line": 195, + "start_line": 128, + "end_line": 197, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -66963,14 +67972,14 @@ } ], "anchor_syntax": "region", - "body": " # #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]\n # @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.\n # @PRE sql is valid SQL string. database_id is a valid Superset DB ID.\n # @POST Returns execution result dict with query_id and status.\n # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.\n def execute_sql(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_sql\"):\n client = self._get_client()\n db_id = database_id or self._database_id\n if db_id is None:\n db_id = self.resolve_database_id()\n # Handle UUID string passed as database_id — resolve to numeric ID\n if isinstance(db_id, str):\n try:\n db_id = int(db_id)\n except (ValueError, TypeError):\n db_id = self.resolve_database_uuid(db_id)\n\n logger.reason(\"Submitting SQL to Superset SQL Lab\", {\n \"database_id\": db_id,\n \"sql_length\": len(sql),\n })\n\n payload = {\n \"database_id\": db_id,\n \"sql\": sql,\n \"client_id\": uuid.uuid4().hex[:10],\n \"schema\": None,\n }\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/sqllab/execute/\",\n data=json.dumps(payload),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n logger.explore(\"SQL Lab execute failed\", {\"error\": str(e)})\n raise ValueError(f\"Superset SQL Lab execute failed: {e}\")\n\n # Parse response — try multiple formats\n result = response if isinstance(response, dict) else {}\n query_id = (result.get(\"query_id\")\n or result.get(\"id\")\n or result.get(\"sql_lab_session_ref\")\n or (result.get(\"result\") or {}).get(\"query_id\")\n or (result.get(\"result\") or {}).get(\"id\"))\n status = result.get(\"status\", \"unknown\")\n\n logger.reason(\"SQL Lab execute response\", {\n \"query_id\": query_id,\n \"status\": status,\n \"has_query_id\": query_id is not None,\n \"has_errors\": \"errors\" in result or \"error\" in result,\n \"response_keys\": list(result.keys()) if isinstance(result, dict) else type(result).__name__,\n \"response_preview\": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],\n })\n\n return {\n \"query_id\": query_id,\n \"status\": status,\n \"raw_response\": result,\n \"database_id\": db_id,\n }\n # #endregion execute_sql\n" + "body": " # #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute]\n # @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info.\n # @PRE sql is valid SQL string. database_id is a valid Superset DB ID.\n # @POST Returns execution result dict with query_id and status.\n # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.\n def execute_sql(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_sql\"):\n client = self._get_client()\n db_id = database_id or self._database_id\n if db_id is None:\n db_id = self.resolve_database_id()\n # Handle UUID string passed as database_id — resolve to numeric ID\n if isinstance(db_id, str):\n try:\n db_id = int(db_id)\n except (ValueError, TypeError):\n db_id = self.resolve_database_uuid(db_id)\n\n log.reason(\"Submitting SQL to Superset SQL Lab\", payload={\n \"database_id\": db_id,\n \"sql_length\": len(sql),\n })\n\n payload = {\n \"database_id\": db_id,\n \"sql\": sql,\n \"client_id\": uuid.uuid4().hex[:10],\n \"schema\": None,\n }\n\n try:\n response = client.network.request(\n method=\"POST\",\n endpoint=\"/api/v1/sqllab/execute/\",\n data=json.dumps(payload),\n headers={\"Content-Type\": \"application/json\"},\n )\n except Exception as e:\n log.explore(\"SQL Lab execute failed\", error=str(e))\n raise ValueError(f\"Superset SQL Lab execute failed: {e}\")\n\n # Parse response — try multiple formats\n result = response if isinstance(response, dict) else {}\n query_id = (result.get(\"query_id\")\n or result.get(\"id\")\n or result.get(\"sql_lab_session_ref\")\n or (result.get(\"result\") or {}).get(\"query_id\")\n or (result.get(\"result\") or {}).get(\"id\"))\n status = result.get(\"status\", \"unknown\")\n\n log.reason(\"SQL Lab execute response\", payload={\n \"query_id\": query_id,\n \"status\": status,\n \"has_query_id\": query_id is not None,\n \"has_errors\": \"errors\" in result or \"error\" in result,\n \"response_keys\": list(result.keys()) if isinstance(result, dict) else type(result).__name__,\n \"response_preview\": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000],\n })\n\n return {\n \"query_id\": query_id,\n \"status\": status,\n \"raw_response\": result,\n \"database_id\": db_id,\n }\n # #endregion execute_sql\n" }, { "contract_id": "poll_execution_status", "contract_type": "Function", "file_path": "backend/src/plugins/translate/superset_executor.py", - "start_line": 197, - "end_line": 286, + "start_line": 199, + "end_line": 290, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -67008,14 +68017,14 @@ } ], "anchor_syntax": "region", - "body": " # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]\n # @BRIEF Poll Superset for SQL execution status until completion or timeout.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns final execution status dict with success/failure/timeout.\n # @SIDE_EFFECT Makes HTTP GET requests to Superset API.\n def poll_execution_status(\n self,\n query_id: str,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.poll_execution_status\"):\n client = self._get_client()\n\n logger.reason(\"Polling SQL execution status\", {\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n \"interval\": poll_interval_seconds,\n })\n\n for attempt in range(max_polls):\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}\",\n )\n result = response if isinstance(response, dict) else {}\n status = result.get(\"status\", \"unknown\")\n state = result.get(\"state\", result.get(\"status\", \"\"))\n\n # Terminal states\n if state in (\"success\", \"finished\", \"completed\"):\n logger.reason(\"SQL execution completed\", {\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"rows_affected\": result.get(\"rows\"),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"success\",\n \"state\": state,\n \"rows_affected\": result.get(\"rows\"),\n \"error_message\": result.get(\"error_message\"),\n \"results\": result.get(\"results\"),\n \"completed_on\": result.get(\"completed_on\"),\n }\n\n if state in (\"failed\", \"error\", \"stopped\"):\n error_msg = result.get(\"error_message\", \"Unknown error\")\n logger.explore(\"SQL execution failed\", {\n \"query_id\": query_id,\n \"state\": state,\n \"error\": error_msg,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"state\": state,\n \"error_message\": error_msg,\n \"results\": None,\n }\n\n if state in (\"pending\", \"running\", \"started\"):\n time.sleep(poll_interval_seconds)\n continue\n\n # Unknown state — treat as still running\n time.sleep(poll_interval_seconds)\n\n except Exception as e:\n logger.explore(\"Polling error, retrying\", {\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"error\": str(e),\n })\n time.sleep(poll_interval_seconds)\n continue\n\n # Timeout\n logger.explore(\"SQL execution polling timed out\", {\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"timeout\",\n \"error_message\": f\"Polling timed out after {max_polls} attempts\",\n \"results\": None,\n }\n # #endregion poll_execution_status\n" + "body": " # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status]\n # @BRIEF Poll Superset for SQL execution status until completion or timeout.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns final execution status dict with success/failure/timeout.\n # @SIDE_EFFECT Makes HTTP GET requests to Superset API.\n def poll_execution_status(\n self,\n query_id: str,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.poll_execution_status\"):\n client = self._get_client()\n\n log.reason(\"Polling SQL execution status\", payload={\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n \"interval\": poll_interval_seconds,\n })\n\n for attempt in range(max_polls):\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}\",\n )\n result = response if isinstance(response, dict) else {}\n status = result.get(\"status\", \"unknown\")\n state = result.get(\"state\", result.get(\"status\", \"\"))\n\n # Terminal states\n if state in (\"success\", \"finished\", \"completed\"):\n log.reason(\"SQL execution completed\", payload={\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"rows_affected\": result.get(\"rows\"),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"success\",\n \"state\": state,\n \"rows_affected\": result.get(\"rows\"),\n \"error_message\": result.get(\"error_message\"),\n \"results\": result.get(\"results\"),\n \"completed_on\": result.get(\"completed_on\"),\n }\n\n if state in (\"failed\", \"error\", \"stopped\"):\n error_msg = result.get(\"error_message\", \"Unknown error\")\n log.explore(\"SQL execution failed\", error=\"SQL query execution returned failed/error state\",\n payload={\n \"query_id\": query_id,\n \"state\": state,\n \"error\": error_msg,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"state\": state,\n \"error_message\": error_msg,\n \"results\": None,\n }\n\n if state in (\"pending\", \"running\", \"started\"):\n time.sleep(poll_interval_seconds)\n continue\n\n # Unknown state — treat as still running\n time.sleep(poll_interval_seconds)\n\n except Exception as e:\n log.explore(\"Polling error, retrying\", error=\"Polling encountered error, will retry\",\n payload={\n \"query_id\": query_id,\n \"attempt\": attempt + 1,\n \"error\": str(e),\n })\n time.sleep(poll_interval_seconds)\n continue\n\n # Timeout\n log.explore(\"SQL execution polling timed out\", error=\"Polling timed out\", payload={\n \"query_id\": query_id,\n \"max_polls\": max_polls,\n })\n return {\n \"query_id\": query_id,\n \"status\": \"timeout\",\n \"error_message\": f\"Polling timed out after {max_polls} attempts\",\n \"results\": None,\n }\n # #endregion poll_execution_status\n" }, { "contract_id": "execute_and_poll", "contract_type": "Function", "file_path": "backend/src/plugins/translate/superset_executor.py", - "start_line": 288, - "end_line": 322, + "start_line": 292, + "end_line": 326, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -67053,14 +68062,14 @@ } ], "anchor_syntax": "region", - "body": " # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]\n # @BRIEF Execute SQL and wait for completion. One-shot convenience method.\n # @PRE sql is valid SQL.\n # @POST Returns final execution result.\n # @SIDE_EFFECT Makes HTTP calls to Superset API.\n def execute_and_poll(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_and_poll\"):\n exec_result = self.execute_sql(\n sql=sql,\n database_id=database_id,\n )\n\n query_id = exec_result.get(\"query_id\")\n if not query_id:\n logger.explore(\"No query_id from SQL Lab execute\", {\n \"raw_response\": exec_result.get(\"raw_response\"),\n })\n return {\n \"status\": \"failed\",\n \"error_message\": \"No query_id returned from SQL Lab\",\n \"query_id\": None,\n }\n\n return self.poll_execution_status(\n query_id=str(query_id),\n max_polls=max_polls,\n poll_interval_seconds=poll_interval_seconds,\n )\n # #endregion execute_and_poll\n" + "body": " # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll]\n # @BRIEF Execute SQL and wait for completion. One-shot convenience method.\n # @PRE sql is valid SQL.\n # @POST Returns final execution result.\n # @SIDE_EFFECT Makes HTTP calls to Superset API.\n def execute_and_poll(\n self,\n sql: str,\n database_id: Optional[Union[int, str]] = None,\n max_polls: int = 60,\n poll_interval_seconds: float = 2.0,\n ) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.execute_and_poll\"):\n exec_result = self.execute_sql(\n sql=sql,\n database_id=database_id,\n )\n\n query_id = exec_result.get(\"query_id\")\n if not query_id:\n log.explore(\"No query_id from SQL Lab execute\", error=\"No query_id returned\", payload={\n \"raw_response\": exec_result.get(\"raw_response\"),\n })\n return {\n \"status\": \"failed\",\n \"error_message\": \"No query_id returned from SQL Lab\",\n \"query_id\": None,\n }\n\n return self.poll_execution_status(\n query_id=str(query_id),\n max_polls=max_polls,\n poll_interval_seconds=poll_interval_seconds,\n )\n # #endregion execute_and_poll\n" }, { "contract_id": "get_query_results", "contract_type": "Function", "file_path": "backend/src/plugins/translate/superset_executor.py", - "start_line": 324, - "end_line": 354, + "start_line": 328, + "end_line": 359, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -67098,7 +68107,7 @@ } ], "anchor_syntax": "region", - "body": " # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]\n # @BRIEF Fetch the results of a completed query from Superset.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns query results if available, or error dict.\n # @SIDE_EFFECT Makes HTTP GET to Superset API.\n def get_query_results(self, query_id: str) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.get_query_results\"):\n client = self._get_client()\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}/results\",\n )\n result = response if isinstance(response, dict) else {}\n return {\n \"query_id\": query_id,\n \"status\": \"success\" if result.get(\"results\") else \"no_results\",\n \"results\": result.get(\"results\"),\n }\n except Exception as e:\n logger.explore(\"Failed to fetch query results\", {\n \"query_id\": query_id,\n \"error\": str(e),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"error_message\": str(e),\n \"results\": None,\n }\n # #endregion get_query_results\n" + "body": " # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results]\n # @BRIEF Fetch the results of a completed query from Superset.\n # @PRE query_id is a valid Superset query ID.\n # @POST Returns query results if available, or error dict.\n # @SIDE_EFFECT Makes HTTP GET to Superset API.\n def get_query_results(self, query_id: str) -> Dict[str, Any]:\n with belief_scope(\"SupersetSqlLabExecutor.get_query_results\"):\n client = self._get_client()\n try:\n response = client.network.request(\n method=\"GET\",\n endpoint=f\"/api/v1/query/{query_id}/results\",\n )\n result = response if isinstance(response, dict) else {}\n return {\n \"query_id\": query_id,\n \"status\": \"success\" if result.get(\"results\") else \"no_results\",\n \"results\": result.get(\"results\"),\n }\n except Exception as e:\n log.explore(\"Failed to fetch query results\", error=\"Failed to fetch query results from Superset\",\n payload={\n \"query_id\": query_id,\n \"error\": str(e),\n })\n return {\n \"query_id\": query_id,\n \"status\": \"failed\",\n \"error_message\": str(e),\n \"results\": None,\n }\n # #endregion get_query_results\n" }, { "contract_id": "SchemasPackage", @@ -70251,7 +71260,7 @@ "contract_type": "Module", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", "start_line": 1, - "end_line": 402, + "end_line": 401, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -70411,14 +71420,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:MigrateSqliteToPostgresScript:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: migration, sqlite, postgresql, config, task_logs, task_records\n# @PURPOSE: Migrates legacy config and task history from SQLite/file storage to PostgreSQL.\n# @LAYER: Scripts\n# @RELATION: READS_FROM -> backend/tasks.db\n# @RELATION: READS_FROM -> backend/config.json\n# @RELATION: WRITES_TO -> postgresql.task_records\n# @RELATION: WRITES_TO -> postgresql.task_logs\n# @RELATION: WRITES_TO -> postgresql.app_configurations\n#\n# @INVARIANT: Script is idempotent for task_records and app_configurations.\n\n# [SECTION: IMPORTS]\nimport argparse\nimport json\nimport os\nimport sqlite3\nfrom pathlib import Path\nfrom typing import Any, Dict, Iterable, Optional\n\nfrom sqlalchemy import create_engine, text\nfrom sqlalchemy.exc import SQLAlchemyError\n\nfrom src.core.logger import belief_scope, logger\n# [/SECTION]\n\n\n# [DEF:Constants:Section]\nDEFAULT_TARGET_URL = os.getenv(\n \"DATABASE_URL\",\n os.getenv(\n \"POSTGRES_URL\",\n \"postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools\",\n ),\n)\n# [/DEF:Constants:Section]\n\n\n# [DEF:_json_load_if_needed:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Parses JSON-like values from SQLite TEXT/JSON columns to Python objects.\n# @PRE: value is scalar JSON/text/list/dict or None.\n# @POST: Returns normalized Python object or original scalar value.\ndef _json_load_if_needed(value: Any) -> Any:\n with belief_scope(\"_json_load_if_needed\"):\n if value is None:\n return None\n if isinstance(value, (dict, list)):\n return value\n if isinstance(value, str):\n raw = value.strip()\n if not raw:\n return None\n if raw[0] in \"{[\":\n try:\n return json.loads(raw)\n except json.JSONDecodeError:\n return value\n return value\n\n\n# [/DEF:_json_load_if_needed:Function]\n\n\n# [DEF:_find_legacy_config_path:Function]\n# @PURPOSE: Resolves the existing legacy config.json path from candidates.\ndef _find_legacy_config_path(explicit_path: Optional[str]) -> Optional[Path]:\n with belief_scope(\"_find_legacy_config_path\"):\n if explicit_path:\n p = Path(explicit_path)\n return p if p.exists() else None\n\n candidates = [\n Path(\"backend/config.json\"),\n Path(\"config.json\"),\n ]\n for candidate in candidates:\n if candidate.exists():\n return candidate\n return None\n\n\n# [/DEF:_find_legacy_config_path:Function]\n\n\n# [DEF:_connect_sqlite:Function]\n# @PURPOSE: Opens a SQLite connection with row factory.\ndef _connect_sqlite(path: Path) -> sqlite3.Connection:\n with belief_scope(\"_connect_sqlite\"):\n conn = sqlite3.connect(str(path))\n conn.row_factory = sqlite3.Row\n return conn\n\n\n# [/DEF:_connect_sqlite:Function]\n\n\n# [DEF:_ensure_target_schema:Function]\n# @PURPOSE: Ensures required PostgreSQL tables exist before migration.\ndef _ensure_target_schema(engine) -> None:\n with belief_scope(\"_ensure_target_schema\"):\n stmts: Iterable[str] = (\n \"\"\"\n CREATE TABLE IF NOT EXISTS app_configurations (\n id TEXT PRIMARY KEY,\n payload JSONB NOT NULL,\n updated_at TIMESTAMPTZ DEFAULT NOW()\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS task_records (\n id TEXT PRIMARY KEY,\n type TEXT NOT NULL,\n status TEXT NOT NULL,\n environment_id TEXT NULL,\n started_at TIMESTAMPTZ NULL,\n finished_at TIMESTAMPTZ NULL,\n logs JSONB NULL,\n error TEXT NULL,\n result JSONB NULL,\n created_at TIMESTAMPTZ DEFAULT NOW(),\n params JSONB NULL\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS task_logs (\n id INTEGER PRIMARY KEY,\n task_id TEXT NOT NULL,\n timestamp TIMESTAMPTZ NOT NULL,\n level VARCHAR(16) NOT NULL,\n source VARCHAR(64) NOT NULL DEFAULT 'system',\n message TEXT NOT NULL,\n metadata_json TEXT NULL,\n CONSTRAINT fk_task_logs_task\n FOREIGN KEY(task_id)\n REFERENCES task_records(id)\n ON DELETE CASCADE\n )\n \"\"\",\n \"CREATE INDEX IF NOT EXISTS ix_task_logs_task_timestamp ON task_logs (task_id, timestamp)\",\n \"CREATE INDEX IF NOT EXISTS ix_task_logs_task_level ON task_logs (task_id, level)\",\n \"CREATE INDEX IF NOT EXISTS ix_task_logs_task_source ON task_logs (task_id, source)\",\n \"\"\"\n DO $$\n BEGIN\n IF EXISTS (\n SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'task_logs_id_seq'\n ) THEN\n PERFORM 1;\n ELSE\n CREATE SEQUENCE task_logs_id_seq OWNED BY task_logs.id;\n END IF;\n END $$;\n \"\"\",\n \"ALTER TABLE task_logs ALTER COLUMN id SET DEFAULT nextval('task_logs_id_seq')\",\n )\n with engine.begin() as conn:\n for stmt in stmts:\n conn.execute(text(stmt))\n\n\n# [/DEF:_ensure_target_schema:Function]\n\n\n# [DEF:_migrate_config:Function]\n# @PURPOSE: Migrates legacy config.json into app_configurations(global).\ndef _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:\n with belief_scope(\"_migrate_config\"):\n if legacy_config_path is None:\n logger.info(\n \"[_migrate_config][Action] No legacy config.json found, skipping\"\n )\n return 0\n\n payload = json.loads(legacy_config_path.read_text(encoding=\"utf-8\"))\n with engine.begin() as conn:\n conn.execute(\n text(\n \"\"\"\n INSERT INTO app_configurations (id, payload, updated_at)\n VALUES ('global', CAST(:payload AS JSONB), NOW())\n ON CONFLICT (id)\n DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW()\n \"\"\"\n ),\n {\"payload\": json.dumps(payload, ensure_ascii=True)},\n )\n logger.info(\n \"[_migrate_config][Coherence:OK] Config migrated from %s\",\n legacy_config_path,\n )\n return 1\n\n\n# [/DEF:_migrate_config:Function]\n\n\n# [DEF:_migrate_tasks_and_logs:Function]\n# @PURPOSE: Migrates task_records and task_logs from SQLite into PostgreSQL.\ndef _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str, int]:\n with belief_scope(\"_migrate_tasks_and_logs\"):\n stats = {\n \"task_records_total\": 0,\n \"task_records_inserted\": 0,\n \"task_logs_total\": 0,\n \"task_logs_inserted\": 0,\n }\n\n rows = sqlite_conn.execute(\n \"\"\"\n SELECT id, type, status, environment_id, started_at, finished_at, logs, error, result, created_at, params\n FROM task_records\n ORDER BY created_at ASC\n \"\"\"\n ).fetchall()\n stats[\"task_records_total\"] = len(rows)\n\n with engine.begin() as conn:\n existing_env_ids = {\n row[0]\n for row in conn.execute(text(\"SELECT id FROM environments\")).fetchall()\n }\n for row in rows:\n params_obj = _json_load_if_needed(row[\"params\"])\n result_obj = _json_load_if_needed(row[\"result\"])\n logs_obj = _json_load_if_needed(row[\"logs\"])\n environment_id = row[\"environment_id\"]\n if environment_id and environment_id not in existing_env_ids:\n # Legacy task may reference environments that were not migrated; keep task row and drop FK value.\n environment_id = None\n\n res = conn.execute(\n text(\n \"\"\"\n INSERT INTO task_records (\n id, type, status, environment_id, started_at, finished_at,\n logs, error, result, created_at, params\n ) VALUES (\n :id, :type, :status, :environment_id, :started_at, :finished_at,\n CAST(:logs AS JSONB), :error, CAST(:result AS JSONB), :created_at, CAST(:params AS JSONB)\n )\n ON CONFLICT (id) DO NOTHING\n \"\"\"\n ),\n {\n \"id\": row[\"id\"],\n \"type\": row[\"type\"],\n \"status\": row[\"status\"],\n \"environment_id\": environment_id,\n \"started_at\": row[\"started_at\"],\n \"finished_at\": row[\"finished_at\"],\n \"logs\": json.dumps(logs_obj, ensure_ascii=True)\n if logs_obj is not None\n else None,\n \"error\": row[\"error\"],\n \"result\": json.dumps(result_obj, ensure_ascii=True)\n if result_obj is not None\n else None,\n \"created_at\": row[\"created_at\"],\n \"params\": json.dumps(params_obj, ensure_ascii=True)\n if params_obj is not None\n else None,\n },\n )\n if res.rowcount and res.rowcount > 0:\n stats[\"task_records_inserted\"] += int(res.rowcount)\n\n log_rows = sqlite_conn.execute(\n \"\"\"\n SELECT id, task_id, timestamp, level, source, message, metadata_json\n FROM task_logs\n ORDER BY id ASC\n \"\"\"\n ).fetchall()\n stats[\"task_logs_total\"] = len(log_rows)\n\n with engine.begin() as conn:\n for row in log_rows:\n # Preserve original IDs to keep migration idempotent.\n res = conn.execute(\n text(\n \"\"\"\n INSERT INTO task_logs (id, task_id, timestamp, level, source, message, metadata_json)\n VALUES (:id, :task_id, :timestamp, :level, :source, :message, :metadata_json)\n ON CONFLICT (id) DO NOTHING\n \"\"\"\n ),\n {\n \"id\": row[\"id\"],\n \"task_id\": row[\"task_id\"],\n \"timestamp\": row[\"timestamp\"],\n \"level\": row[\"level\"],\n \"source\": row[\"source\"] or \"system\",\n \"message\": row[\"message\"],\n \"metadata_json\": row[\"metadata_json\"],\n },\n )\n if res.rowcount and res.rowcount > 0:\n stats[\"task_logs_inserted\"] += int(res.rowcount)\n\n # Ensure sequence is aligned after explicit id inserts.\n conn.execute(\n text(\n \"\"\"\n SELECT setval(\n 'task_logs_id_seq',\n COALESCE((SELECT MAX(id) FROM task_logs), 1),\n TRUE\n )\n \"\"\"\n )\n )\n\n logger.info(\n \"[_migrate_tasks_and_logs][Coherence:OK] task_records=%s/%s task_logs=%s/%s\",\n stats[\"task_records_inserted\"],\n stats[\"task_records_total\"],\n stats[\"task_logs_inserted\"],\n stats[\"task_logs_total\"],\n )\n return stats\n\n\n# [/DEF:_migrate_tasks_and_logs:Function]\n\n\n# [DEF:run_migration:Function]\n# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL.\ndef run_migration(\n sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]\n) -> Dict[str, int]:\n with belief_scope(\"run_migration\"):\n logger.info(\n \"[run_migration][Entry] sqlite=%s target=%s\", sqlite_path, target_url\n )\n if not sqlite_path.exists():\n raise FileNotFoundError(f\"SQLite source not found: {sqlite_path}\")\n\n sqlite_conn = _connect_sqlite(sqlite_path)\n engine = create_engine(target_url, pool_pre_ping=True)\n try:\n _ensure_target_schema(engine)\n config_upserted = _migrate_config(engine, legacy_config_path)\n stats = _migrate_tasks_and_logs(engine, sqlite_conn)\n stats[\"config_upserted\"] = config_upserted\n return stats\n finally:\n sqlite_conn.close()\n\n\n# [/DEF:run_migration:Function]\n\n\n# [DEF:main:Function]\n# @PURPOSE: CLI entrypoint.\ndef main() -> int:\n with belief_scope(\"main\"):\n parser = argparse.ArgumentParser(\n description=\"Migrate legacy config.json and task logs from SQLite to PostgreSQL.\",\n )\n parser.add_argument(\n \"--sqlite-path\",\n default=\"backend/tasks.db\",\n help=\"Path to source SQLite DB with task_records/task_logs (default: backend/tasks.db).\",\n )\n parser.add_argument(\n \"--target-url\",\n default=DEFAULT_TARGET_URL,\n help=\"Target PostgreSQL SQLAlchemy URL (default: DATABASE_URL/POSTGRES_URL env).\",\n )\n parser.add_argument(\n \"--config-path\",\n default=None,\n help=\"Optional path to legacy config.json (auto-detected when omitted).\",\n )\n\n args = parser.parse_args()\n\n sqlite_path = Path(args.sqlite_path)\n legacy_config_path = _find_legacy_config_path(args.config_path)\n try:\n stats = run_migration(\n sqlite_path=sqlite_path,\n target_url=args.target_url,\n legacy_config_path=legacy_config_path,\n )\n print(\"Migration completed.\")\n print(json.dumps(stats, indent=2))\n return 0\n except (SQLAlchemyError, OSError, sqlite3.Error, ValueError) as e:\n logger.error(\"[main][Coherence:Failed] Migration failed: %s\", e)\n print(f\"Migration failed: {e}\")\n return 1\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n# [/DEF:main:Function]\n\n# [/DEF:MigrateSqliteToPostgresScript:Module]\n" + "body": "# [DEF:MigrateSqliteToPostgresScript:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: migration, sqlite, postgresql, config, task_logs, task_records\n# @PURPOSE: Migrates legacy config and task history from SQLite/file storage to PostgreSQL.\n# @LAYER: Scripts\n# @RELATION: READS_FROM -> backend/tasks.db\n# @RELATION: READS_FROM -> backend/config.json\n# @RELATION: WRITES_TO -> postgresql.task_records\n# @RELATION: WRITES_TO -> postgresql.task_logs\n# @RELATION: WRITES_TO -> postgresql.app_configurations\n#\n# @INVARIANT: Script is idempotent for task_records and app_configurations.\n\n# [SECTION: IMPORTS]\nimport argparse\nimport json\nimport os\nimport sqlite3\nfrom pathlib import Path\nfrom typing import Any, Dict, Iterable, Optional\n\nfrom sqlalchemy import create_engine, text\nfrom sqlalchemy.exc import SQLAlchemyError\n\nfrom src.core.logger import belief_scope\nfrom src.core.cot_logger import MarkerLogger\nlog = MarkerLogger(\"MigrateSqliteToPostgres\")\n\n# [/SECTION]\n\n\n# [DEF:Constants:Section]\nDEFAULT_TARGET_URL = os.getenv(\n \"DATABASE_URL\",\n os.getenv(\n \"POSTGRES_URL\",\n \"postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools\",\n ),\n)\n# [/DEF:Constants:Section]\n\n\n# [DEF:_json_load_if_needed:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Parses JSON-like values from SQLite TEXT/JSON columns to Python objects.\n# @PRE: value is scalar JSON/text/list/dict or None.\n# @POST: Returns normalized Python object or original scalar value.\ndef _json_load_if_needed(value: Any) -> Any:\n with belief_scope(\"_json_load_if_needed\"):\n if value is None:\n return None\n if isinstance(value, (dict, list)):\n return value\n if isinstance(value, str):\n raw = value.strip()\n if not raw:\n return None\n if raw[0] in \"{[\":\n try:\n return json.loads(raw)\n except json.JSONDecodeError:\n return value\n return value\n\n\n# [/DEF:_json_load_if_needed:Function]\n\n\n# [DEF:_find_legacy_config_path:Function]\n# @PURPOSE: Resolves the existing legacy config.json path from candidates.\ndef _find_legacy_config_path(explicit_path: Optional[str]) -> Optional[Path]:\n with belief_scope(\"_find_legacy_config_path\"):\n if explicit_path:\n p = Path(explicit_path)\n return p if p.exists() else None\n\n candidates = [\n Path(\"backend/config.json\"),\n Path(\"config.json\"),\n ]\n for candidate in candidates:\n if candidate.exists():\n return candidate\n return None\n\n\n# [/DEF:_find_legacy_config_path:Function]\n\n\n# [DEF:_connect_sqlite:Function]\n# @PURPOSE: Opens a SQLite connection with row factory.\ndef _connect_sqlite(path: Path) -> sqlite3.Connection:\n with belief_scope(\"_connect_sqlite\"):\n conn = sqlite3.connect(str(path))\n conn.row_factory = sqlite3.Row\n return conn\n\n\n# [/DEF:_connect_sqlite:Function]\n\n\n# [DEF:_ensure_target_schema:Function]\n# @PURPOSE: Ensures required PostgreSQL tables exist before migration.\ndef _ensure_target_schema(engine) -> None:\n with belief_scope(\"_ensure_target_schema\"):\n stmts: Iterable[str] = (\n \"\"\"\n CREATE TABLE IF NOT EXISTS app_configurations (\n id TEXT PRIMARY KEY,\n payload JSONB NOT NULL,\n updated_at TIMESTAMPTZ DEFAULT NOW()\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS task_records (\n id TEXT PRIMARY KEY,\n type TEXT NOT NULL,\n status TEXT NOT NULL,\n environment_id TEXT NULL,\n started_at TIMESTAMPTZ NULL,\n finished_at TIMESTAMPTZ NULL,\n logs JSONB NULL,\n error TEXT NULL,\n result JSONB NULL,\n created_at TIMESTAMPTZ DEFAULT NOW(),\n params JSONB NULL\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS task_logs (\n id INTEGER PRIMARY KEY,\n task_id TEXT NOT NULL,\n timestamp TIMESTAMPTZ NOT NULL,\n level VARCHAR(16) NOT NULL,\n source VARCHAR(64) NOT NULL DEFAULT 'system',\n message TEXT NOT NULL,\n metadata_json TEXT NULL,\n CONSTRAINT fk_task_logs_task\n FOREIGN KEY(task_id)\n REFERENCES task_records(id)\n ON DELETE CASCADE\n )\n \"\"\",\n \"CREATE INDEX IF NOT EXISTS ix_task_logs_task_timestamp ON task_logs (task_id, timestamp)\",\n \"CREATE INDEX IF NOT EXISTS ix_task_logs_task_level ON task_logs (task_id, level)\",\n \"CREATE INDEX IF NOT EXISTS ix_task_logs_task_source ON task_logs (task_id, source)\",\n \"\"\"\n DO $$\n BEGIN\n IF EXISTS (\n SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'task_logs_id_seq'\n ) THEN\n PERFORM 1;\n ELSE\n CREATE SEQUENCE task_logs_id_seq OWNED BY task_logs.id;\n END IF;\n END $$;\n \"\"\",\n \"ALTER TABLE task_logs ALTER COLUMN id SET DEFAULT nextval('task_logs_id_seq')\",\n )\n with engine.begin() as conn:\n for stmt in stmts:\n conn.execute(text(stmt))\n\n\n# [/DEF:_ensure_target_schema:Function]\n\n\n# [DEF:_migrate_config:Function]\n# @PURPOSE: Migrates legacy config.json into app_configurations(global).\ndef _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:\n with belief_scope(\"_migrate_config\"):\n if legacy_config_path is None:\n log.reason(\"No legacy config.json found, skipping migration\")\n return 0\n\n payload = json.loads(legacy_config_path.read_text(encoding=\"utf-8\"))\n with engine.begin() as conn:\n conn.execute(\n text(\n \"\"\"\n INSERT INTO app_configurations (id, payload, updated_at)\n VALUES ('global', CAST(:payload AS JSONB), NOW())\n ON CONFLICT (id)\n DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW()\n \"\"\"\n ),\n {\"payload\": json.dumps(payload, ensure_ascii=True)},\n )\n logger.info(\n \"[_migrate_config][Coherence:OK] Config migrated from %s\",\n legacy_config_path,\n )\n return 1\n\n\n# [/DEF:_migrate_config:Function]\n\n\n# [DEF:_migrate_tasks_and_logs:Function]\n# @PURPOSE: Migrates task_records and task_logs from SQLite into PostgreSQL.\ndef _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str, int]:\n with belief_scope(\"_migrate_tasks_and_logs\"):\n stats = {\n \"task_records_total\": 0,\n \"task_records_inserted\": 0,\n \"task_logs_total\": 0,\n \"task_logs_inserted\": 0,\n }\n\n rows = sqlite_conn.execute(\n \"\"\"\n SELECT id, type, status, environment_id, started_at, finished_at, logs, error, result, created_at, params\n FROM task_records\n ORDER BY created_at ASC\n \"\"\"\n ).fetchall()\n stats[\"task_records_total\"] = len(rows)\n\n with engine.begin() as conn:\n existing_env_ids = {\n row[0]\n for row in conn.execute(text(\"SELECT id FROM environments\")).fetchall()\n }\n for row in rows:\n params_obj = _json_load_if_needed(row[\"params\"])\n result_obj = _json_load_if_needed(row[\"result\"])\n logs_obj = _json_load_if_needed(row[\"logs\"])\n environment_id = row[\"environment_id\"]\n if environment_id and environment_id not in existing_env_ids:\n # Legacy task may reference environments that were not migrated; keep task row and drop FK value.\n environment_id = None\n\n res = conn.execute(\n text(\n \"\"\"\n INSERT INTO task_records (\n id, type, status, environment_id, started_at, finished_at,\n logs, error, result, created_at, params\n ) VALUES (\n :id, :type, :status, :environment_id, :started_at, :finished_at,\n CAST(:logs AS JSONB), :error, CAST(:result AS JSONB), :created_at, CAST(:params AS JSONB)\n )\n ON CONFLICT (id) DO NOTHING\n \"\"\"\n ),\n {\n \"id\": row[\"id\"],\n \"type\": row[\"type\"],\n \"status\": row[\"status\"],\n \"environment_id\": environment_id,\n \"started_at\": row[\"started_at\"],\n \"finished_at\": row[\"finished_at\"],\n \"logs\": json.dumps(logs_obj, ensure_ascii=True)\n if logs_obj is not None\n else None,\n \"error\": row[\"error\"],\n \"result\": json.dumps(result_obj, ensure_ascii=True)\n if result_obj is not None\n else None,\n \"created_at\": row[\"created_at\"],\n \"params\": json.dumps(params_obj, ensure_ascii=True)\n if params_obj is not None\n else None,\n },\n )\n if res.rowcount and res.rowcount > 0:\n stats[\"task_records_inserted\"] += int(res.rowcount)\n\n log_rows = sqlite_conn.execute(\n \"\"\"\n SELECT id, task_id, timestamp, level, source, message, metadata_json\n FROM task_logs\n ORDER BY id ASC\n \"\"\"\n ).fetchall()\n stats[\"task_logs_total\"] = len(log_rows)\n\n with engine.begin() as conn:\n for row in log_rows:\n # Preserve original IDs to keep migration idempotent.\n res = conn.execute(\n text(\n \"\"\"\n INSERT INTO task_logs (id, task_id, timestamp, level, source, message, metadata_json)\n VALUES (:id, :task_id, :timestamp, :level, :source, :message, :metadata_json)\n ON CONFLICT (id) DO NOTHING\n \"\"\"\n ),\n {\n \"id\": row[\"id\"],\n \"task_id\": row[\"task_id\"],\n \"timestamp\": row[\"timestamp\"],\n \"level\": row[\"level\"],\n \"source\": row[\"source\"] or \"system\",\n \"message\": row[\"message\"],\n \"metadata_json\": row[\"metadata_json\"],\n },\n )\n if res.rowcount and res.rowcount > 0:\n stats[\"task_logs_inserted\"] += int(res.rowcount)\n\n # Ensure sequence is aligned after explicit id inserts.\n conn.execute(\n text(\n \"\"\"\n SELECT setval(\n 'task_logs_id_seq',\n COALESCE((SELECT MAX(id) FROM task_logs), 1),\n TRUE\n )\n \"\"\"\n )\n )\n\n logger.info(\n \"[_migrate_tasks_and_logs][Coherence:OK] task_records=%s/%s task_logs=%s/%s\",\n stats[\"task_records_inserted\"],\n stats[\"task_records_total\"],\n stats[\"task_logs_inserted\"],\n stats[\"task_logs_total\"],\n )\n return stats\n\n\n# [/DEF:_migrate_tasks_and_logs:Function]\n\n\n# [DEF:run_migration:Function]\n# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL.\ndef run_migration(\n sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]\n) -> Dict[str, int]:\n with belief_scope(\"run_migration\"):\n log.reason(\"Starting migration from SQLite to PostgreSQL\", payload={\"sqlite_path\": str(sqlite_path), \"target_url\": target_url})\n if not sqlite_path.exists():\n raise FileNotFoundError(f\"SQLite source not found: {sqlite_path}\")\n\n sqlite_conn = _connect_sqlite(sqlite_path)\n engine = create_engine(target_url, pool_pre_ping=True)\n try:\n _ensure_target_schema(engine)\n config_upserted = _migrate_config(engine, legacy_config_path)\n stats = _migrate_tasks_and_logs(engine, sqlite_conn)\n stats[\"config_upserted\"] = config_upserted\n return stats\n finally:\n sqlite_conn.close()\n\n\n# [/DEF:run_migration:Function]\n\n\n# [DEF:main:Function]\n# @PURPOSE: CLI entrypoint.\ndef main() -> int:\n with belief_scope(\"main\"):\n parser = argparse.ArgumentParser(\n description=\"Migrate legacy config.json and task logs from SQLite to PostgreSQL.\",\n )\n parser.add_argument(\n \"--sqlite-path\",\n default=\"backend/tasks.db\",\n help=\"Path to source SQLite DB with task_records/task_logs (default: backend/tasks.db).\",\n )\n parser.add_argument(\n \"--target-url\",\n default=DEFAULT_TARGET_URL,\n help=\"Target PostgreSQL SQLAlchemy URL (default: DATABASE_URL/POSTGRES_URL env).\",\n )\n parser.add_argument(\n \"--config-path\",\n default=None,\n help=\"Optional path to legacy config.json (auto-detected when omitted).\",\n )\n\n args = parser.parse_args()\n\n sqlite_path = Path(args.sqlite_path)\n legacy_config_path = _find_legacy_config_path(args.config_path)\n try:\n stats = run_migration(\n sqlite_path=sqlite_path,\n target_url=args.target_url,\n legacy_config_path=legacy_config_path,\n )\n print(\"Migration completed.\")\n print(json.dumps(stats, indent=2))\n return 0\n except (SQLAlchemyError, OSError, sqlite3.Error, ValueError) as e:\n logger.error(\"[main][Coherence:Failed] Migration failed: %s\", e)\n print(f\"Migration failed: {e}\")\n return 1\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n# [/DEF:main:Function]\n\n# [/DEF:MigrateSqliteToPostgresScript:Module]\n" }, { "contract_id": "Constants", "contract_type": "Section", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 30, - "end_line": 38, + "start_line": 33, + "end_line": 41, "tier": "TIER_1", "complexity": 1, "metadata": {}, @@ -70431,8 +71440,8 @@ "contract_id": "_json_load_if_needed", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 41, - "end_line": 64, + "start_line": 44, + "end_line": 67, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -70478,8 +71487,8 @@ "contract_id": "_find_legacy_config_path", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 67, - "end_line": 85, + "start_line": 70, + "end_line": 88, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70494,8 +71503,8 @@ "contract_id": "_connect_sqlite", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 88, - "end_line": 97, + "start_line": 91, + "end_line": 100, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70510,8 +71519,8 @@ "contract_id": "_ensure_target_schema", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 100, - "end_line": 164, + "start_line": 103, + "end_line": 167, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70526,8 +71535,8 @@ "contract_id": "_migrate_config", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 167, - "end_line": 197, + "start_line": 170, + "end_line": 198, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70536,14 +71545,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:_migrate_config:Function]\n# @PURPOSE: Migrates legacy config.json into app_configurations(global).\ndef _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:\n with belief_scope(\"_migrate_config\"):\n if legacy_config_path is None:\n logger.info(\n \"[_migrate_config][Action] No legacy config.json found, skipping\"\n )\n return 0\n\n payload = json.loads(legacy_config_path.read_text(encoding=\"utf-8\"))\n with engine.begin() as conn:\n conn.execute(\n text(\n \"\"\"\n INSERT INTO app_configurations (id, payload, updated_at)\n VALUES ('global', CAST(:payload AS JSONB), NOW())\n ON CONFLICT (id)\n DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW()\n \"\"\"\n ),\n {\"payload\": json.dumps(payload, ensure_ascii=True)},\n )\n logger.info(\n \"[_migrate_config][Coherence:OK] Config migrated from %s\",\n legacy_config_path,\n )\n return 1\n\n\n# [/DEF:_migrate_config:Function]\n" + "body": "# [DEF:_migrate_config:Function]\n# @PURPOSE: Migrates legacy config.json into app_configurations(global).\ndef _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:\n with belief_scope(\"_migrate_config\"):\n if legacy_config_path is None:\n log.reason(\"No legacy config.json found, skipping migration\")\n return 0\n\n payload = json.loads(legacy_config_path.read_text(encoding=\"utf-8\"))\n with engine.begin() as conn:\n conn.execute(\n text(\n \"\"\"\n INSERT INTO app_configurations (id, payload, updated_at)\n VALUES ('global', CAST(:payload AS JSONB), NOW())\n ON CONFLICT (id)\n DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW()\n \"\"\"\n ),\n {\"payload\": json.dumps(payload, ensure_ascii=True)},\n )\n logger.info(\n \"[_migrate_config][Coherence:OK] Config migrated from %s\",\n legacy_config_path,\n )\n return 1\n\n\n# [/DEF:_migrate_config:Function]\n" }, { "contract_id": "_migrate_tasks_and_logs", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 200, - "end_line": 326, + "start_line": 201, + "end_line": 327, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70558,8 +71567,8 @@ "contract_id": "run_migration", "contract_type": "Function", "file_path": "backend/src/scripts/migrate_sqlite_to_postgres.py", - "start_line": 329, - "end_line": 353, + "start_line": 330, + "end_line": 352, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70568,7 +71577,7 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:run_migration:Function]\n# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL.\ndef run_migration(\n sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]\n) -> Dict[str, int]:\n with belief_scope(\"run_migration\"):\n logger.info(\n \"[run_migration][Entry] sqlite=%s target=%s\", sqlite_path, target_url\n )\n if not sqlite_path.exists():\n raise FileNotFoundError(f\"SQLite source not found: {sqlite_path}\")\n\n sqlite_conn = _connect_sqlite(sqlite_path)\n engine = create_engine(target_url, pool_pre_ping=True)\n try:\n _ensure_target_schema(engine)\n config_upserted = _migrate_config(engine, legacy_config_path)\n stats = _migrate_tasks_and_logs(engine, sqlite_conn)\n stats[\"config_upserted\"] = config_upserted\n return stats\n finally:\n sqlite_conn.close()\n\n\n# [/DEF:run_migration:Function]\n" + "body": "# [DEF:run_migration:Function]\n# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL.\ndef run_migration(\n sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]\n) -> Dict[str, int]:\n with belief_scope(\"run_migration\"):\n log.reason(\"Starting migration from SQLite to PostgreSQL\", payload={\"sqlite_path\": str(sqlite_path), \"target_url\": target_url})\n if not sqlite_path.exists():\n raise FileNotFoundError(f\"SQLite source not found: {sqlite_path}\")\n\n sqlite_conn = _connect_sqlite(sqlite_path)\n engine = create_engine(target_url, pool_pre_ping=True)\n try:\n _ensure_target_schema(engine)\n config_upserted = _migrate_config(engine, legacy_config_path)\n stats = _migrate_tasks_and_logs(engine, sqlite_conn)\n stats[\"config_upserted\"] = config_upserted\n return stats\n finally:\n sqlite_conn.close()\n\n\n# [/DEF:run_migration:Function]\n" }, { "contract_id": "SeedPermissionsScript", @@ -70792,7 +71801,7 @@ "contract_type": "Module", "file_path": "backend/src/scripts/seed_superset_load_test.py", "start_line": 1, - "end_line": 399, + "end_line": 394, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -70883,14 +71892,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SeedSupersetLoadTestScript:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: superset, load-test, charts, dashboards, seed, stress\n# @PURPOSE: Creates randomized load-test data in Superset by cloning chart configurations and creating dashboards in target environments.\n# @LAYER: Scripts\n# @RELATION: USES -> [ConfigManager]\n# @RELATION: USES -> [SupersetClient]\n# @INVARIANT: Created chart and dashboard names are globally unique for one script run.\n\n# [SECTION: IMPORTS]\nimport argparse\nimport json\nimport random\nimport sys\nimport uuid\nfrom pathlib import Path\nfrom typing import Dict, List, Optional\n\nsys.path.append(str(Path(__file__).parent.parent.parent))\n\nfrom src.core.config_manager import ConfigManager\nfrom src.core.config_models import Environment\nfrom src.core.logger import belief_scope, logger\nfrom src.core.superset_client import SupersetClient\n# [/SECTION]\n\n\n# [DEF:_parse_args:Function]\n# @PURPOSE: Parses CLI arguments for load-test data generation.\n# @PRE: Script is called from CLI.\n# @POST: Returns validated argument namespace.\ndef _parse_args() -> argparse.Namespace:\n parser = argparse.ArgumentParser(\n description=\"Seed Superset with load-test charts and dashboards\"\n )\n parser.add_argument(\n \"--envs\", nargs=\"+\", default=[\"ss1\", \"ss2\"], help=\"Target environment IDs\"\n )\n parser.add_argument(\n \"--charts\", type=int, default=10000, help=\"Target number of charts to create\"\n )\n parser.add_argument(\n \"--dashboards\",\n type=int,\n default=500,\n help=\"Target number of dashboards to create\",\n )\n parser.add_argument(\n \"--template-pool-size\",\n type=int,\n default=200,\n help=\"How many source charts to sample as templates per env\",\n )\n parser.add_argument(\n \"--seed\", type=int, default=None, help=\"Optional RNG seed for reproducibility\"\n )\n parser.add_argument(\n \"--max-errors\",\n type=int,\n default=100,\n help=\"Stop early if errors exceed this threshold\",\n )\n parser.add_argument(\n \"--dry-run\", action=\"store_true\", help=\"Do not write data, only validate setup\"\n )\n return parser.parse_args()\n\n\n# [/DEF:_parse_args:Function]\n\n\n# [DEF:_extract_result_payload:Function]\n# @PURPOSE: Normalizes Superset API payloads that may be wrapped in `result`.\n# @PRE: payload is a JSON-decoded API response.\n# @POST: Returns the unwrapped object when present.\ndef _extract_result_payload(payload: Dict) -> Dict:\n result = payload.get(\"result\")\n if isinstance(result, dict):\n return result\n return payload\n\n\n# [/DEF:_extract_result_payload:Function]\n\n\n# [DEF:_extract_created_id:Function]\n# @PURPOSE: Extracts object ID from create/update API response.\n# @PRE: payload is a JSON-decoded API response.\n# @POST: Returns integer object ID or None if missing.\ndef _extract_created_id(payload: Dict) -> Optional[int]:\n direct_id = payload.get(\"id\")\n if isinstance(direct_id, int):\n return direct_id\n result = payload.get(\"result\")\n if isinstance(result, dict) and isinstance(result.get(\"id\"), int):\n return int(result[\"id\"])\n return None\n\n\n# [/DEF:_extract_created_id:Function]\n\n\n# [DEF:_generate_unique_name:Function]\n# @PURPOSE: Generates globally unique random names for charts/dashboards.\n# @PRE: used_names is mutable set for collision tracking.\n# @POST: Returns a unique string and stores it in used_names.\ndef _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) -> str:\n adjectives = [\n \"amber\",\n \"rapid\",\n \"frozen\",\n \"delta\",\n \"lunar\",\n \"vector\",\n \"cobalt\",\n \"silent\",\n \"neon\",\n \"solar\",\n ]\n nouns = [\n \"falcon\",\n \"matrix\",\n \"signal\",\n \"harbor\",\n \"stream\",\n \"vertex\",\n \"bridge\",\n \"orbit\",\n \"pulse\",\n \"forge\",\n ]\n while True:\n token = uuid.uuid4().hex[:8]\n candidate = f\"{prefix}_{rng.choice(adjectives)}_{rng.choice(nouns)}_{rng.randint(100, 999)}_{token}\"\n if candidate not in used_names:\n used_names.add(candidate)\n return candidate\n\n\n# [/DEF:_generate_unique_name:Function]\n\n\n# [DEF:_resolve_target_envs:Function]\n# @PURPOSE: Resolves requested environment IDs from configuration.\n# @PRE: env_ids is non-empty.\n# @POST: Returns mapping env_id -> configured environment object.\ndef _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:\n config_manager = ConfigManager()\n configured = {env.id: env for env in config_manager.get_environments()}\n resolved: Dict[str, Environment] = {}\n\n if not configured:\n for config_path in [Path(\"config.json\"), Path(\"backend/config.json\")]:\n if not config_path.exists():\n continue\n try:\n payload = json.loads(config_path.read_text(encoding=\"utf-8\"))\n env_rows = payload.get(\"environments\", [])\n for row in env_rows:\n env = Environment(**row)\n configured[env.id] = env\n except Exception as exc:\n logger.warning(\n f\"[REFLECT] Failed loading environments from {config_path}: {exc}\"\n )\n\n for env_id in env_ids:\n env = configured.get(env_id)\n if env is None:\n raise ValueError(f\"Environment '{env_id}' not found in configuration\")\n resolved[env_id] = env\n\n return resolved\n\n\n# [/DEF:_resolve_target_envs:Function]\n\n\n# [DEF:_build_chart_template_pool:Function]\n# @PURPOSE: Builds a pool of source chart templates to clone in one environment.\n# @PRE: Client is authenticated.\n# @POST: Returns non-empty list of chart payload templates.\ndef _build_chart_template_pool(\n client: SupersetClient, pool_size: int, rng: random.Random\n) -> List[Dict]:\n list_query = {\n \"page\": 0,\n \"page_size\": 1000,\n \"columns\": [\n \"id\",\n \"slice_name\",\n \"datasource_id\",\n \"datasource_type\",\n \"viz_type\",\n \"params\",\n \"query_context\",\n ],\n }\n rows = client.network.fetch_paginated_data(\n endpoint=\"/chart/\",\n pagination_options={\"base_query\": list_query, \"results_field\": \"result\"},\n )\n\n candidates = [row for row in rows if isinstance(row, dict) and row.get(\"id\")]\n if not candidates:\n raise RuntimeError(\"No source charts available for templating\")\n\n selected = (\n candidates\n if len(candidates) <= pool_size\n else rng.sample(candidates, pool_size)\n )\n templates: List[Dict] = []\n\n for row in selected:\n chart_id = int(row[\"id\"])\n detail_payload = client.get_chart(chart_id)\n detail = _extract_result_payload(detail_payload)\n\n datasource_id = detail.get(\"datasource_id\")\n datasource_type = (\n detail.get(\"datasource_type\") or row.get(\"datasource_type\") or \"table\"\n )\n if datasource_id is None:\n continue\n\n params_value = detail.get(\"params\")\n if isinstance(params_value, dict):\n params_value = json.dumps(params_value)\n\n query_context_value = detail.get(\"query_context\")\n if isinstance(query_context_value, dict):\n query_context_value = json.dumps(query_context_value)\n\n templates.append(\n {\n \"datasource_id\": int(datasource_id),\n \"datasource_type\": str(datasource_type),\n \"viz_type\": detail.get(\"viz_type\") or row.get(\"viz_type\"),\n \"params\": params_value,\n \"query_context\": query_context_value,\n }\n )\n\n if not templates:\n raise RuntimeError(\"Could not build templates with datasource metadata\")\n\n return templates\n\n\n# [/DEF:_build_chart_template_pool:Function]\n\n\n# [DEF:seed_superset_load_data:Function]\n# @PURPOSE: Creates dashboards and cloned charts for load testing across target environments.\n# @PRE: Target environments must be reachable and authenticated.\n# @POST: Returns execution statistics dictionary.\n# @SIDE_EFFECT: Creates objects in Superset environments.\ndef seed_superset_load_data(args: argparse.Namespace) -> Dict:\n rng = random.Random(args.seed)\n env_map = _resolve_target_envs(args.envs)\n\n clients: Dict[str, SupersetClient] = {}\n templates_by_env: Dict[str, List[Dict]] = {}\n created_dashboards: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n created_charts: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n used_chart_names: set[str] = set()\n used_dashboard_names: set[str] = set()\n\n for env_id, env in env_map.items():\n client = SupersetClient(env)\n client.authenticate()\n clients[env_id] = client\n templates_by_env[env_id] = _build_chart_template_pool(\n client, args.template_pool_size, rng\n )\n logger.info(\n f\"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates\"\n )\n\n errors = 0\n env_ids = list(env_map.keys())\n\n for idx in range(args.dashboards):\n env_id = (\n env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)\n )\n dashboard_title = _generate_unique_name(\"lt_dash\", used_dashboard_names, rng)\n\n if args.dry_run:\n logger.info(\n f\"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}\"\n )\n continue\n\n try:\n payload = {\"dashboard_title\": dashboard_title, \"published\": False}\n created = clients[env_id].network.request(\n \"POST\", \"/dashboard/\", data=json.dumps(payload)\n )\n dashboard_id = _extract_created_id(created)\n if dashboard_id is None:\n raise RuntimeError(f\"Dashboard create response missing id: {created}\")\n created_dashboards[env_id].append(dashboard_id)\n except Exception as exc:\n errors += 1\n logger.error(f\"[EXPLORE] Failed creating dashboard in {env_id}: {exc}\")\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n if args.dry_run:\n return {\n \"dry_run\": True,\n \"templates_by_env\": {k: len(v) for k, v in templates_by_env.items()},\n \"charts_target\": args.charts,\n \"dashboards_target\": args.dashboards,\n }\n\n for env_id in env_ids:\n if not created_dashboards[env_id]:\n raise RuntimeError(\n f\"No dashboards created in environment {env_id}; cannot bind charts\"\n )\n\n for index in range(args.charts):\n env_id = rng.choice(env_ids)\n client = clients[env_id]\n template = rng.choice(templates_by_env[env_id])\n dashboard_id = rng.choice(created_dashboards[env_id])\n chart_name = _generate_unique_name(\"lt_chart\", used_chart_names, rng)\n\n payload = {\n \"slice_name\": chart_name,\n \"datasource_id\": template[\"datasource_id\"],\n \"datasource_type\": template[\"datasource_type\"],\n \"dashboards\": [dashboard_id],\n }\n if template.get(\"viz_type\"):\n payload[\"viz_type\"] = template[\"viz_type\"]\n if template.get(\"params\"):\n payload[\"params\"] = template[\"params\"]\n if template.get(\"query_context\"):\n payload[\"query_context\"] = template[\"query_context\"]\n\n try:\n created = client.network.request(\n \"POST\", \"/chart/\", data=json.dumps(payload)\n )\n chart_id = _extract_created_id(created)\n if chart_id is None:\n raise RuntimeError(f\"Chart create response missing id: {created}\")\n created_charts[env_id].append(chart_id)\n\n if (index + 1) % 500 == 0:\n logger.info(f\"[REASON] Created {index + 1}/{args.charts} charts\")\n except Exception as exc:\n errors += 1\n logger.error(f\"[EXPLORE] Failed creating chart in {env_id}: {exc}\")\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n return {\n \"dry_run\": False,\n \"errors\": errors,\n \"dashboards\": {env_id: len(ids) for env_id, ids in created_dashboards.items()},\n \"charts\": {env_id: len(ids) for env_id, ids in created_charts.items()},\n \"total_dashboards\": sum(len(ids) for ids in created_dashboards.values()),\n \"total_charts\": sum(len(ids) for ids in created_charts.values()),\n }\n\n\n# [/DEF:seed_superset_load_data:Function]\n\n\n# [DEF:main:Function]\n# @PURPOSE: CLI entrypoint for Superset load-test data seeding.\n# @PRE: Command line arguments are valid.\n# @POST: Prints summary and exits with non-zero status on failure.\ndef main() -> None:\n with belief_scope(\"seed_superset_load_test.main\"):\n args = _parse_args()\n result = seed_superset_load_data(args)\n logger.info(\n f\"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}\"\n )\n\n\n# [/DEF:main:Function]\n\n\nif __name__ == \"__main__\":\n main()\n\n# [/DEF:SeedSupersetLoadTestScript:Module]\n" + "body": "# [DEF:SeedSupersetLoadTestScript:Module]\n#\n# @COMPLEXITY: 3\n# @SEMANTICS: superset, load-test, charts, dashboards, seed, stress\n# @PURPOSE: Creates randomized load-test data in Superset by cloning chart configurations and creating dashboards in target environments.\n# @LAYER: Scripts\n# @RELATION: USES -> [ConfigManager]\n# @RELATION: USES -> [SupersetClient]\n# @INVARIANT: Created chart and dashboard names are globally unique for one script run.\n\n# [SECTION: IMPORTS]\nimport argparse\nimport json\nimport random\nimport sys\nimport uuid\nfrom pathlib import Path\nfrom typing import Dict, List, Optional\n\nsys.path.append(str(Path(__file__).parent.parent.parent))\n\nfrom src.core.config_manager import ConfigManager\nfrom src.core.config_models import Environment\nfrom src.core.logger import belief_scope\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.superset_client import SupersetClient\nlog = MarkerLogger(\"SeedSupersetLoadTest\")\n\n# [/SECTION]\n\n\n# [DEF:_parse_args:Function]\n# @PURPOSE: Parses CLI arguments for load-test data generation.\n# @PRE: Script is called from CLI.\n# @POST: Returns validated argument namespace.\ndef _parse_args() -> argparse.Namespace:\n parser = argparse.ArgumentParser(\n description=\"Seed Superset with load-test charts and dashboards\"\n )\n parser.add_argument(\n \"--envs\", nargs=\"+\", default=[\"ss1\", \"ss2\"], help=\"Target environment IDs\"\n )\n parser.add_argument(\n \"--charts\", type=int, default=10000, help=\"Target number of charts to create\"\n )\n parser.add_argument(\n \"--dashboards\",\n type=int,\n default=500,\n help=\"Target number of dashboards to create\",\n )\n parser.add_argument(\n \"--template-pool-size\",\n type=int,\n default=200,\n help=\"How many source charts to sample as templates per env\",\n )\n parser.add_argument(\n \"--seed\", type=int, default=None, help=\"Optional RNG seed for reproducibility\"\n )\n parser.add_argument(\n \"--max-errors\",\n type=int,\n default=100,\n help=\"Stop early if errors exceed this threshold\",\n )\n parser.add_argument(\n \"--dry-run\", action=\"store_true\", help=\"Do not write data, only validate setup\"\n )\n return parser.parse_args()\n\n\n# [/DEF:_parse_args:Function]\n\n\n# [DEF:_extract_result_payload:Function]\n# @PURPOSE: Normalizes Superset API payloads that may be wrapped in `result`.\n# @PRE: payload is a JSON-decoded API response.\n# @POST: Returns the unwrapped object when present.\ndef _extract_result_payload(payload: Dict) -> Dict:\n result = payload.get(\"result\")\n if isinstance(result, dict):\n return result\n return payload\n\n\n# [/DEF:_extract_result_payload:Function]\n\n\n# [DEF:_extract_created_id:Function]\n# @PURPOSE: Extracts object ID from create/update API response.\n# @PRE: payload is a JSON-decoded API response.\n# @POST: Returns integer object ID or None if missing.\ndef _extract_created_id(payload: Dict) -> Optional[int]:\n direct_id = payload.get(\"id\")\n if isinstance(direct_id, int):\n return direct_id\n result = payload.get(\"result\")\n if isinstance(result, dict) and isinstance(result.get(\"id\"), int):\n return int(result[\"id\"])\n return None\n\n\n# [/DEF:_extract_created_id:Function]\n\n\n# [DEF:_generate_unique_name:Function]\n# @PURPOSE: Generates globally unique random names for charts/dashboards.\n# @PRE: used_names is mutable set for collision tracking.\n# @POST: Returns a unique string and stores it in used_names.\ndef _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) -> str:\n adjectives = [\n \"amber\",\n \"rapid\",\n \"frozen\",\n \"delta\",\n \"lunar\",\n \"vector\",\n \"cobalt\",\n \"silent\",\n \"neon\",\n \"solar\",\n ]\n nouns = [\n \"falcon\",\n \"matrix\",\n \"signal\",\n \"harbor\",\n \"stream\",\n \"vertex\",\n \"bridge\",\n \"orbit\",\n \"pulse\",\n \"forge\",\n ]\n while True:\n token = uuid.uuid4().hex[:8]\n candidate = f\"{prefix}_{rng.choice(adjectives)}_{rng.choice(nouns)}_{rng.randint(100, 999)}_{token}\"\n if candidate not in used_names:\n used_names.add(candidate)\n return candidate\n\n\n# [/DEF:_generate_unique_name:Function]\n\n\n# [DEF:_resolve_target_envs:Function]\n# @PURPOSE: Resolves requested environment IDs from configuration.\n# @PRE: env_ids is non-empty.\n# @POST: Returns mapping env_id -> configured environment object.\ndef _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:\n config_manager = ConfigManager()\n configured = {env.id: env for env in config_manager.get_environments()}\n resolved: Dict[str, Environment] = {}\n\n if not configured:\n for config_path in [Path(\"config.json\"), Path(\"backend/config.json\")]:\n if not config_path.exists():\n continue\n try:\n payload = json.loads(config_path.read_text(encoding=\"utf-8\"))\n env_rows = payload.get(\"environments\", [])\n for row in env_rows:\n env = Environment(**row)\n configured[env.id] = env\n except Exception as exc:\n log.explore(\"Failed loading environments from config\", payload={\"config_path\": str(config_path)}, error=str(exc))\n\n for env_id in env_ids:\n env = configured.get(env_id)\n if env is None:\n raise ValueError(f\"Environment '{env_id}' not found in configuration\")\n resolved[env_id] = env\n\n return resolved\n\n\n# [/DEF:_resolve_target_envs:Function]\n\n\n# [DEF:_build_chart_template_pool:Function]\n# @PURPOSE: Builds a pool of source chart templates to clone in one environment.\n# @PRE: Client is authenticated.\n# @POST: Returns non-empty list of chart payload templates.\ndef _build_chart_template_pool(\n client: SupersetClient, pool_size: int, rng: random.Random\n) -> List[Dict]:\n list_query = {\n \"page\": 0,\n \"page_size\": 1000,\n \"columns\": [\n \"id\",\n \"slice_name\",\n \"datasource_id\",\n \"datasource_type\",\n \"viz_type\",\n \"params\",\n \"query_context\",\n ],\n }\n rows = client.network.fetch_paginated_data(\n endpoint=\"/chart/\",\n pagination_options={\"base_query\": list_query, \"results_field\": \"result\"},\n )\n\n candidates = [row for row in rows if isinstance(row, dict) and row.get(\"id\")]\n if not candidates:\n raise RuntimeError(\"No source charts available for templating\")\n\n selected = (\n candidates\n if len(candidates) <= pool_size\n else rng.sample(candidates, pool_size)\n )\n templates: List[Dict] = []\n\n for row in selected:\n chart_id = int(row[\"id\"])\n detail_payload = client.get_chart(chart_id)\n detail = _extract_result_payload(detail_payload)\n\n datasource_id = detail.get(\"datasource_id\")\n datasource_type = (\n detail.get(\"datasource_type\") or row.get(\"datasource_type\") or \"table\"\n )\n if datasource_id is None:\n continue\n\n params_value = detail.get(\"params\")\n if isinstance(params_value, dict):\n params_value = json.dumps(params_value)\n\n query_context_value = detail.get(\"query_context\")\n if isinstance(query_context_value, dict):\n query_context_value = json.dumps(query_context_value)\n\n templates.append(\n {\n \"datasource_id\": int(datasource_id),\n \"datasource_type\": str(datasource_type),\n \"viz_type\": detail.get(\"viz_type\") or row.get(\"viz_type\"),\n \"params\": params_value,\n \"query_context\": query_context_value,\n }\n )\n\n if not templates:\n raise RuntimeError(\"Could not build templates with datasource metadata\")\n\n return templates\n\n\n# [/DEF:_build_chart_template_pool:Function]\n\n\n# [DEF:seed_superset_load_data:Function]\n# @PURPOSE: Creates dashboards and cloned charts for load testing across target environments.\n# @PRE: Target environments must be reachable and authenticated.\n# @POST: Returns execution statistics dictionary.\n# @SIDE_EFFECT: Creates objects in Superset environments.\ndef seed_superset_load_data(args: argparse.Namespace) -> Dict:\n rng = random.Random(args.seed)\n env_map = _resolve_target_envs(args.envs)\n\n clients: Dict[str, SupersetClient] = {}\n templates_by_env: Dict[str, List[Dict]] = {}\n created_dashboards: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n created_charts: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n used_chart_names: set[str] = set()\n used_dashboard_names: set[str] = set()\n\n for env_id, env in env_map.items():\n client = SupersetClient(env)\n client.authenticate()\n clients[env_id] = client\n templates_by_env[env_id] = _build_chart_template_pool(\n client, args.template_pool_size, rng\n )\n log.reason(\"Loaded chart templates for environment\", payload={\"env_id\": env_id, \"template_count\": len(templates_by_env[env_id])})\n\n errors = 0\n env_ids = list(env_map.keys())\n\n for idx in range(args.dashboards):\n env_id = (\n env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)\n )\n dashboard_title = _generate_unique_name(\"lt_dash\", used_dashboard_names, rng)\n\n if args.dry_run:\n log.reflect(\"Dry-run dashboard create\", payload={\"env_id\": env_id, \"title\": dashboard_title})\n continue\n\n try:\n payload = {\"dashboard_title\": dashboard_title, \"published\": False}\n created = clients[env_id].network.request(\n \"POST\", \"/dashboard/\", data=json.dumps(payload)\n )\n dashboard_id = _extract_created_id(created)\n if dashboard_id is None:\n raise RuntimeError(f\"Dashboard create response missing id: {created}\")\n created_dashboards[env_id].append(dashboard_id)\n except Exception as exc:\n errors += 1\n log.explore(\"Failed creating dashboard\", payload={\"env_id\": env_id}, error=str(exc))\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n if args.dry_run:\n return {\n \"dry_run\": True,\n \"templates_by_env\": {k: len(v) for k, v in templates_by_env.items()},\n \"charts_target\": args.charts,\n \"dashboards_target\": args.dashboards,\n }\n\n for env_id in env_ids:\n if not created_dashboards[env_id]:\n raise RuntimeError(\n f\"No dashboards created in environment {env_id}; cannot bind charts\"\n )\n\n for index in range(args.charts):\n env_id = rng.choice(env_ids)\n client = clients[env_id]\n template = rng.choice(templates_by_env[env_id])\n dashboard_id = rng.choice(created_dashboards[env_id])\n chart_name = _generate_unique_name(\"lt_chart\", used_chart_names, rng)\n\n payload = {\n \"slice_name\": chart_name,\n \"datasource_id\": template[\"datasource_id\"],\n \"datasource_type\": template[\"datasource_type\"],\n \"dashboards\": [dashboard_id],\n }\n if template.get(\"viz_type\"):\n payload[\"viz_type\"] = template[\"viz_type\"]\n if template.get(\"params\"):\n payload[\"params\"] = template[\"params\"]\n if template.get(\"query_context\"):\n payload[\"query_context\"] = template[\"query_context\"]\n\n try:\n created = client.network.request(\n \"POST\", \"/chart/\", data=json.dumps(payload)\n )\n chart_id = _extract_created_id(created)\n if chart_id is None:\n raise RuntimeError(f\"Chart create response missing id: {created}\")\n created_charts[env_id].append(chart_id)\n\n if (index + 1) % 500 == 0:\n log.reason(f\"Created {index + 1}/{args.charts} charts\")\n except Exception as exc:\n errors += 1\n log.explore(\"Failed creating chart\", payload={\"env_id\": env_id}, error=str(exc))\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n return {\n \"dry_run\": False,\n \"errors\": errors,\n \"dashboards\": {env_id: len(ids) for env_id, ids in created_dashboards.items()},\n \"charts\": {env_id: len(ids) for env_id, ids in created_charts.items()},\n \"total_dashboards\": sum(len(ids) for ids in created_dashboards.values()),\n \"total_charts\": sum(len(ids) for ids in created_charts.values()),\n }\n\n\n# [/DEF:seed_superset_load_data:Function]\n\n\n# [DEF:main:Function]\n# @PURPOSE: CLI entrypoint for Superset load-test data seeding.\n# @PRE: Command line arguments are valid.\n# @POST: Prints summary and exits with non-zero status on failure.\ndef main() -> None:\n with belief_scope(\"seed_superset_load_test.main\"):\n args = _parse_args()\n result = seed_superset_load_data(args)\n log.reflect(\"Seed load test complete\", payload={\"summary\": result})\n\n\n# [/DEF:main:Function]\n\n\nif __name__ == \"__main__\":\n main()\n\n# [/DEF:SeedSupersetLoadTestScript:Module]\n" }, { "contract_id": "_parse_args", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 29, - "end_line": 70, + "start_line": 32, + "end_line": 73, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70926,8 +71935,8 @@ "contract_id": "_extract_result_payload", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 73, - "end_line": 84, + "start_line": 76, + "end_line": 87, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -70963,8 +71972,8 @@ "contract_id": "_extract_created_id", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 87, - "end_line": 101, + "start_line": 90, + "end_line": 104, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -71000,8 +72009,8 @@ "contract_id": "_generate_unique_name", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 104, - "end_line": 141, + "start_line": 107, + "end_line": 144, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -71037,8 +72046,8 @@ "contract_id": "_resolve_target_envs", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 144, - "end_line": 177, + "start_line": 147, + "end_line": 178, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -71068,14 +72077,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:_resolve_target_envs:Function]\n# @PURPOSE: Resolves requested environment IDs from configuration.\n# @PRE: env_ids is non-empty.\n# @POST: Returns mapping env_id -> configured environment object.\ndef _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:\n config_manager = ConfigManager()\n configured = {env.id: env for env in config_manager.get_environments()}\n resolved: Dict[str, Environment] = {}\n\n if not configured:\n for config_path in [Path(\"config.json\"), Path(\"backend/config.json\")]:\n if not config_path.exists():\n continue\n try:\n payload = json.loads(config_path.read_text(encoding=\"utf-8\"))\n env_rows = payload.get(\"environments\", [])\n for row in env_rows:\n env = Environment(**row)\n configured[env.id] = env\n except Exception as exc:\n logger.warning(\n f\"[REFLECT] Failed loading environments from {config_path}: {exc}\"\n )\n\n for env_id in env_ids:\n env = configured.get(env_id)\n if env is None:\n raise ValueError(f\"Environment '{env_id}' not found in configuration\")\n resolved[env_id] = env\n\n return resolved\n\n\n# [/DEF:_resolve_target_envs:Function]\n" + "body": "# [DEF:_resolve_target_envs:Function]\n# @PURPOSE: Resolves requested environment IDs from configuration.\n# @PRE: env_ids is non-empty.\n# @POST: Returns mapping env_id -> configured environment object.\ndef _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:\n config_manager = ConfigManager()\n configured = {env.id: env for env in config_manager.get_environments()}\n resolved: Dict[str, Environment] = {}\n\n if not configured:\n for config_path in [Path(\"config.json\"), Path(\"backend/config.json\")]:\n if not config_path.exists():\n continue\n try:\n payload = json.loads(config_path.read_text(encoding=\"utf-8\"))\n env_rows = payload.get(\"environments\", [])\n for row in env_rows:\n env = Environment(**row)\n configured[env.id] = env\n except Exception as exc:\n log.explore(\"Failed loading environments from config\", payload={\"config_path\": str(config_path)}, error=str(exc))\n\n for env_id in env_ids:\n env = configured.get(env_id)\n if env is None:\n raise ValueError(f\"Environment '{env_id}' not found in configuration\")\n resolved[env_id] = env\n\n return resolved\n\n\n# [/DEF:_resolve_target_envs:Function]\n" }, { "contract_id": "_build_chart_template_pool", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 180, - "end_line": 252, + "start_line": 181, + "end_line": 253, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -71111,8 +72120,8 @@ "contract_id": "seed_superset_load_data", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 255, - "end_line": 377, + "start_line": 256, + "end_line": 374, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -71152,14 +72161,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:seed_superset_load_data:Function]\n# @PURPOSE: Creates dashboards and cloned charts for load testing across target environments.\n# @PRE: Target environments must be reachable and authenticated.\n# @POST: Returns execution statistics dictionary.\n# @SIDE_EFFECT: Creates objects in Superset environments.\ndef seed_superset_load_data(args: argparse.Namespace) -> Dict:\n rng = random.Random(args.seed)\n env_map = _resolve_target_envs(args.envs)\n\n clients: Dict[str, SupersetClient] = {}\n templates_by_env: Dict[str, List[Dict]] = {}\n created_dashboards: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n created_charts: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n used_chart_names: set[str] = set()\n used_dashboard_names: set[str] = set()\n\n for env_id, env in env_map.items():\n client = SupersetClient(env)\n client.authenticate()\n clients[env_id] = client\n templates_by_env[env_id] = _build_chart_template_pool(\n client, args.template_pool_size, rng\n )\n logger.info(\n f\"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates\"\n )\n\n errors = 0\n env_ids = list(env_map.keys())\n\n for idx in range(args.dashboards):\n env_id = (\n env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)\n )\n dashboard_title = _generate_unique_name(\"lt_dash\", used_dashboard_names, rng)\n\n if args.dry_run:\n logger.info(\n f\"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}\"\n )\n continue\n\n try:\n payload = {\"dashboard_title\": dashboard_title, \"published\": False}\n created = clients[env_id].network.request(\n \"POST\", \"/dashboard/\", data=json.dumps(payload)\n )\n dashboard_id = _extract_created_id(created)\n if dashboard_id is None:\n raise RuntimeError(f\"Dashboard create response missing id: {created}\")\n created_dashboards[env_id].append(dashboard_id)\n except Exception as exc:\n errors += 1\n logger.error(f\"[EXPLORE] Failed creating dashboard in {env_id}: {exc}\")\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n if args.dry_run:\n return {\n \"dry_run\": True,\n \"templates_by_env\": {k: len(v) for k, v in templates_by_env.items()},\n \"charts_target\": args.charts,\n \"dashboards_target\": args.dashboards,\n }\n\n for env_id in env_ids:\n if not created_dashboards[env_id]:\n raise RuntimeError(\n f\"No dashboards created in environment {env_id}; cannot bind charts\"\n )\n\n for index in range(args.charts):\n env_id = rng.choice(env_ids)\n client = clients[env_id]\n template = rng.choice(templates_by_env[env_id])\n dashboard_id = rng.choice(created_dashboards[env_id])\n chart_name = _generate_unique_name(\"lt_chart\", used_chart_names, rng)\n\n payload = {\n \"slice_name\": chart_name,\n \"datasource_id\": template[\"datasource_id\"],\n \"datasource_type\": template[\"datasource_type\"],\n \"dashboards\": [dashboard_id],\n }\n if template.get(\"viz_type\"):\n payload[\"viz_type\"] = template[\"viz_type\"]\n if template.get(\"params\"):\n payload[\"params\"] = template[\"params\"]\n if template.get(\"query_context\"):\n payload[\"query_context\"] = template[\"query_context\"]\n\n try:\n created = client.network.request(\n \"POST\", \"/chart/\", data=json.dumps(payload)\n )\n chart_id = _extract_created_id(created)\n if chart_id is None:\n raise RuntimeError(f\"Chart create response missing id: {created}\")\n created_charts[env_id].append(chart_id)\n\n if (index + 1) % 500 == 0:\n logger.info(f\"[REASON] Created {index + 1}/{args.charts} charts\")\n except Exception as exc:\n errors += 1\n logger.error(f\"[EXPLORE] Failed creating chart in {env_id}: {exc}\")\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n return {\n \"dry_run\": False,\n \"errors\": errors,\n \"dashboards\": {env_id: len(ids) for env_id, ids in created_dashboards.items()},\n \"charts\": {env_id: len(ids) for env_id, ids in created_charts.items()},\n \"total_dashboards\": sum(len(ids) for ids in created_dashboards.values()),\n \"total_charts\": sum(len(ids) for ids in created_charts.values()),\n }\n\n\n# [/DEF:seed_superset_load_data:Function]\n" + "body": "# [DEF:seed_superset_load_data:Function]\n# @PURPOSE: Creates dashboards and cloned charts for load testing across target environments.\n# @PRE: Target environments must be reachable and authenticated.\n# @POST: Returns execution statistics dictionary.\n# @SIDE_EFFECT: Creates objects in Superset environments.\ndef seed_superset_load_data(args: argparse.Namespace) -> Dict:\n rng = random.Random(args.seed)\n env_map = _resolve_target_envs(args.envs)\n\n clients: Dict[str, SupersetClient] = {}\n templates_by_env: Dict[str, List[Dict]] = {}\n created_dashboards: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n created_charts: Dict[str, List[int]] = {env_id: [] for env_id in env_map}\n used_chart_names: set[str] = set()\n used_dashboard_names: set[str] = set()\n\n for env_id, env in env_map.items():\n client = SupersetClient(env)\n client.authenticate()\n clients[env_id] = client\n templates_by_env[env_id] = _build_chart_template_pool(\n client, args.template_pool_size, rng\n )\n log.reason(\"Loaded chart templates for environment\", payload={\"env_id\": env_id, \"template_count\": len(templates_by_env[env_id])})\n\n errors = 0\n env_ids = list(env_map.keys())\n\n for idx in range(args.dashboards):\n env_id = (\n env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)\n )\n dashboard_title = _generate_unique_name(\"lt_dash\", used_dashboard_names, rng)\n\n if args.dry_run:\n log.reflect(\"Dry-run dashboard create\", payload={\"env_id\": env_id, \"title\": dashboard_title})\n continue\n\n try:\n payload = {\"dashboard_title\": dashboard_title, \"published\": False}\n created = clients[env_id].network.request(\n \"POST\", \"/dashboard/\", data=json.dumps(payload)\n )\n dashboard_id = _extract_created_id(created)\n if dashboard_id is None:\n raise RuntimeError(f\"Dashboard create response missing id: {created}\")\n created_dashboards[env_id].append(dashboard_id)\n except Exception as exc:\n errors += 1\n log.explore(\"Failed creating dashboard\", payload={\"env_id\": env_id}, error=str(exc))\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n if args.dry_run:\n return {\n \"dry_run\": True,\n \"templates_by_env\": {k: len(v) for k, v in templates_by_env.items()},\n \"charts_target\": args.charts,\n \"dashboards_target\": args.dashboards,\n }\n\n for env_id in env_ids:\n if not created_dashboards[env_id]:\n raise RuntimeError(\n f\"No dashboards created in environment {env_id}; cannot bind charts\"\n )\n\n for index in range(args.charts):\n env_id = rng.choice(env_ids)\n client = clients[env_id]\n template = rng.choice(templates_by_env[env_id])\n dashboard_id = rng.choice(created_dashboards[env_id])\n chart_name = _generate_unique_name(\"lt_chart\", used_chart_names, rng)\n\n payload = {\n \"slice_name\": chart_name,\n \"datasource_id\": template[\"datasource_id\"],\n \"datasource_type\": template[\"datasource_type\"],\n \"dashboards\": [dashboard_id],\n }\n if template.get(\"viz_type\"):\n payload[\"viz_type\"] = template[\"viz_type\"]\n if template.get(\"params\"):\n payload[\"params\"] = template[\"params\"]\n if template.get(\"query_context\"):\n payload[\"query_context\"] = template[\"query_context\"]\n\n try:\n created = client.network.request(\n \"POST\", \"/chart/\", data=json.dumps(payload)\n )\n chart_id = _extract_created_id(created)\n if chart_id is None:\n raise RuntimeError(f\"Chart create response missing id: {created}\")\n created_charts[env_id].append(chart_id)\n\n if (index + 1) % 500 == 0:\n log.reason(f\"Created {index + 1}/{args.charts} charts\")\n except Exception as exc:\n errors += 1\n log.explore(\"Failed creating chart\", payload={\"env_id\": env_id}, error=str(exc))\n if errors >= args.max_errors:\n raise RuntimeError(\n f\"Stopping due to max errors reached ({errors})\"\n ) from exc\n\n return {\n \"dry_run\": False,\n \"errors\": errors,\n \"dashboards\": {env_id: len(ids) for env_id, ids in created_dashboards.items()},\n \"charts\": {env_id: len(ids) for env_id, ids in created_charts.items()},\n \"total_dashboards\": sum(len(ids) for ids in created_dashboards.values()),\n \"total_charts\": sum(len(ids) for ids in created_charts.values()),\n }\n\n\n# [/DEF:seed_superset_load_data:Function]\n" }, { "contract_id": "main", "contract_type": "Function", "file_path": "backend/src/scripts/seed_superset_load_test.py", - "start_line": 380, - "end_line": 393, + "start_line": 377, + "end_line": 388, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -71189,7 +72198,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:main:Function]\n# @PURPOSE: CLI entrypoint for Superset load-test data seeding.\n# @PRE: Command line arguments are valid.\n# @POST: Prints summary and exits with non-zero status on failure.\ndef main() -> None:\n with belief_scope(\"seed_superset_load_test.main\"):\n args = _parse_args()\n result = seed_superset_load_data(args)\n logger.info(\n f\"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}\"\n )\n\n\n# [/DEF:main:Function]\n" + "body": "# [DEF:main:Function]\n# @PURPOSE: CLI entrypoint for Superset load-test data seeding.\n# @PRE: Command line arguments are valid.\n# @POST: Prints summary and exits with non-zero status on failure.\ndef main() -> None:\n with belief_scope(\"seed_superset_load_test.main\"):\n args = _parse_args()\n result = seed_superset_load_data(args)\n log.reflect(\"Seed load test complete\", payload={\"summary\": result})\n\n\n# [/DEF:main:Function]\n" }, { "contract_id": "test_dataset_dashboard_relations_script", @@ -80646,7 +81655,7 @@ "contract_type": "Module", "file_path": "backend/src/services/dataset_review/clarification_engine.py", "start_line": 1, - "end_line": 303, + "end_line": 306, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -80728,14 +81737,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:ClarificationEngine:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: dataset_review, clarification, question_payload, answer_persistence, readiness, findings\n# @PURPOSE: Manage one-question-at-a-time clarification state, deterministic answer persistence, and readiness/finding updates.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: DEPENDS_ON -> [ClarificationSession]\n# @RELATION: DEPENDS_ON -> [ClarificationQuestion]\n# @RELATION: DEPENDS_ON -> [ClarificationAnswer]\n# @RELATION: DEPENDS_ON -> [ValidationFinding]\n# @RELATION: DISPATCHES -> [ClarificationHelpers:Module]\n# @PRE: Target session contains a persisted clarification aggregate in the current ownership scope.\n# @POST: Active clarification payload exposes one highest-priority unresolved question, and each recorded answer is persisted before pointer/readiness mutation.\n# @SIDE_EFFECT: Persists clarification answers, question/session states, and related readiness/finding changes.\n# @DATA_CONTRACT: Input[DatasetReviewSession|ClarificationAnswerCommand] -> Output[ClarificationStateResult]\n# @INVARIANT: Only one active clarification question may exist at a time; skipped and expert-review items remain unresolved and visible.\n# @RATIONALE: Original 635-line file exceeded INV_7 (400-line module limit). Extracted pure helpers into _helpers sub-module.\n# @REJECTED: Keeping all clarification logic in one file because it exceeded the fractal limit.\n\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass, field\nfrom datetime import datetime\nfrom typing import List, Optional\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n AnswerKind,\n ClarificationAnswer,\n ClarificationQuestion,\n ClarificationSession,\n ClarificationStatus,\n DatasetReviewSession,\n QuestionState,\n ReadinessState,\n RecommendedAction,\n SessionPhase,\n ValidationFinding,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionRepository,\n)\nfrom src.services.dataset_review.clarification_pkg._helpers import (\n select_next_open_question,\n count_resolved_questions,\n count_remaining_questions,\n normalize_answer_value,\n build_impact_summary,\n upsert_clarification_finding,\n derive_readiness_state,\n derive_recommended_action,\n)\n\n\n# [DEF:ClarificationQuestionPayload:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Typed active-question payload returned to the API layer.\n@dataclass\nclass ClarificationQuestionPayload:\n question_id: str\n clarification_session_id: str\n topic_ref: str\n question_text: str\n why_it_matters: str\n current_guess: Optional[str]\n priority: int\n state: QuestionState\n options: list[dict[str, object]] = field(default_factory=list)\n\n\n# [/DEF:ClarificationQuestionPayload:Class]\n\n\n# [DEF:ClarificationStateResult:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Clarification state result carrying the current session, active payload, and changed findings.\n@dataclass\nclass ClarificationStateResult:\n clarification_session: ClarificationSession\n current_question: Optional[ClarificationQuestionPayload]\n session: DatasetReviewSession\n changed_findings: List[ValidationFinding] = field(default_factory=list)\n\n\n# [/DEF:ClarificationStateResult:Class]\n\n\n# [DEF:ClarificationAnswerCommand:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Typed answer command for clarification state mutation.\n@dataclass\nclass ClarificationAnswerCommand:\n session: DatasetReviewSession\n question_id: str\n answer_kind: AnswerKind\n answer_value: Optional[str]\n user: User\n\n\n# [/DEF:ClarificationAnswerCommand:Class]\n\n\n# [DEF:ClarificationEngine:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Provide deterministic one-question-at-a-time clarification selection and answer persistence.\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: CALLS -> [ClarificationHelpers:Module]\n# @PRE: Repository is bound to the current request transaction scope.\n# @POST: Returned clarification state is persistence-backed and aligned with session readiness/recommended action.\n# @SIDE_EFFECT: Mutates clarification answers, session flags, and related clarification findings.\nclass ClarificationEngine:\n # [DEF:ClarificationEngine_init:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind repository dependency for clarification persistence operations.\n def __init__(self, repository: DatasetReviewSessionRepository) -> None:\n self.repository = repository\n\n # [/DEF:ClarificationEngine_init:Function]\n\n # [DEF:build_question_payload:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Return the one active highest-priority clarification question payload.\n # @PRE: Session contains unresolved clarification state or a resumable clarification session.\n # @POST: Returns exactly one active/open question payload or None when no unresolved question remains.\n # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence.\n def build_question_payload(\n self, session: DatasetReviewSession,\n ) -> Optional[ClarificationQuestionPayload]:\n with belief_scope(\"ClarificationEngine.build_question_payload\"):\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n logger.reason(\"No clarification session found\", extra={\"session_id\": session.session_id})\n return None\n\n active_questions = [\n q for q in clarification_session.questions if q.state == QuestionState.OPEN\n ]\n active_questions.sort(key=lambda item: (-int(item.priority), item.created_at, item.question_id))\n\n if not active_questions:\n clarification_session.current_question_id = None\n clarification_session.status = ClarificationStatus.COMPLETED\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n if session.current_phase == SessionPhase.CLARIFICATION:\n session.current_phase = SessionPhase.REVIEW\n self.repository.db.commit()\n logger.reflect(\"No unresolved clarification question remains\", extra={\"session_id\": session.session_id})\n return None\n\n selected_question = active_questions[0]\n clarification_session.current_question_id = selected_question.question_id\n clarification_session.status = ClarificationStatus.ACTIVE\n session.readiness_state = ReadinessState.CLARIFICATION_ACTIVE\n session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION\n session.current_phase = SessionPhase.CLARIFICATION\n\n logger.reason(\"Selected active clarification question\", extra={\"session_id\": session.session_id, \"question_id\": selected_question.question_id, \"priority\": selected_question.priority})\n self.repository.db.commit()\n\n payload = ClarificationQuestionPayload(\n question_id=selected_question.question_id,\n clarification_session_id=selected_question.clarification_session_id,\n topic_ref=selected_question.topic_ref,\n question_text=selected_question.question_text,\n why_it_matters=selected_question.why_it_matters,\n current_guess=selected_question.current_guess,\n priority=selected_question.priority,\n state=selected_question.state,\n options=[\n {\"option_id\": o.option_id, \"question_id\": o.question_id, \"label\": o.label, \"value\": o.value, \"is_recommended\": o.is_recommended, \"display_order\": o.display_order}\n for o in sorted(selected_question.options, key=lambda item: (item.display_order, item.label, item.option_id))\n ],\n )\n logger.reflect(\"Clarification payload built\", extra={\"session_id\": session.session_id, \"question_id\": payload.question_id, \"option_count\": len(payload.options)})\n return payload\n\n # [/DEF:build_question_payload:Function]\n\n # [DEF:record_answer:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation.\n # @PRE: Target question belongs to the session's active clarification session and is still open.\n # @POST: Answer row is persisted before current-question pointer advances.\n # @SIDE_EFFECT: Inserts answer row, mutates question/session states, updates clarification findings, and commits.\n def record_answer(self, command: ClarificationAnswerCommand) -> ClarificationStateResult:\n with belief_scope(\"ClarificationEngine.record_answer\"):\n session = command.session\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n logger.explore(\"Cannot record clarification answer because no clarification session exists\", extra={\"session_id\": session.session_id})\n raise ValueError(\"Clarification session not found\")\n\n question = self._find_question(clarification_session, command.question_id)\n if question is None:\n logger.explore(\"Cannot record clarification answer for foreign or missing question\", extra={\"session_id\": session.session_id, \"question_id\": command.question_id})\n raise ValueError(\"Clarification question not found\")\n\n if question.answer is not None:\n logger.explore(\"Rejected duplicate clarification answer submission\", extra={\"session_id\": session.session_id, \"question_id\": command.question_id})\n raise ValueError(\"Clarification question already answered\")\n\n if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id:\n logger.explore(\"Rejected answer for non-active clarification question\", extra={\"session_id\": session.session_id, \"question_id\": question.question_id, \"current_question_id\": clarification_session.current_question_id})\n raise ValueError(\"Only the active clarification question can be answered\")\n\n normalized_answer_value = normalize_answer_value(command.answer_kind, command.answer_value, question)\n\n logger.reason(\"Persisting clarification answer before state advancement\", extra={\"session_id\": session.session_id, \"question_id\": question.question_id, \"answer_kind\": command.answer_kind.value})\n persisted_answer = ClarificationAnswer(\n question_id=question.question_id,\n answer_kind=command.answer_kind,\n answer_value=normalized_answer_value,\n answered_by_user_id=command.user.id,\n impact_summary=build_impact_summary(question, command.answer_kind, normalized_answer_value),\n )\n self.repository.db.add(persisted_answer)\n self.repository.db.flush()\n\n changed_finding = upsert_clarification_finding(\n session=session, question=question, answer_kind=command.answer_kind,\n answer_value=normalized_answer_value, db_session=self.repository.db,\n )\n\n if command.answer_kind == AnswerKind.SELECTED:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.CUSTOM:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.SKIPPED:\n question.state = QuestionState.SKIPPED\n elif command.answer_kind == AnswerKind.EXPERT_REVIEW:\n question.state = QuestionState.EXPERT_REVIEW\n\n question.updated_at = datetime.utcnow()\n self.repository.db.flush()\n\n clarification_session.resolved_count = count_resolved_questions(clarification_session)\n clarification_session.remaining_count = count_remaining_questions(clarification_session)\n clarification_session.summary_delta = self.summarize_progress(clarification_session)\n clarification_session.updated_at = datetime.utcnow()\n\n next_question = select_next_open_question(clarification_session)\n clarification_session.current_question_id = next_question.question_id if next_question else None\n clarification_session.status = ClarificationStatus.ACTIVE if next_question else ClarificationStatus.COMPLETED\n if clarification_session.status == ClarificationStatus.COMPLETED:\n clarification_session.completed_at = datetime.utcnow()\n\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n session.current_phase = SessionPhase.CLARIFICATION if clarification_session.current_question_id else SessionPhase.REVIEW\n self.repository.bump_session_version(session)\n\n self.repository.db.commit()\n self.repository.db.refresh(session)\n\n logger.reflect(\"Clarification answer recorded and session advanced\", extra={\"session_id\": session.session_id, \"question_id\": question.question_id, \"next_question_id\": clarification_session.current_question_id, \"readiness_state\": session.readiness_state.value, \"remaining_count\": clarification_session.remaining_count})\n\n return ClarificationStateResult(\n clarification_session=clarification_session,\n current_question=self.build_question_payload(session),\n session=session,\n changed_findings=[changed_finding] if changed_finding else [],\n )\n\n # [/DEF:record_answer:Function]\n\n # [DEF:summarize_progress:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Produce a compact progress summary for pause/resume and completion UX.\n def summarize_progress(self, clarification_session: ClarificationSession) -> str:\n resolved = count_resolved_questions(clarification_session)\n remaining = count_remaining_questions(clarification_session)\n return f\"{resolved} resolved, {remaining} unresolved\"\n\n # [/DEF:summarize_progress:Function]\n\n # [DEF:_get_latest_clarification_session:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Select the latest clarification session for the current dataset review aggregate.\n def _get_latest_clarification_session(self, session: DatasetReviewSession) -> Optional[ClarificationSession]:\n if not session.clarification_sessions:\n return None\n ordered = sorted(session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True)\n return ordered[0]\n\n # [/DEF:_get_latest_clarification_session:Function]\n\n # [DEF:_find_question:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Resolve a clarification question from the active clarification aggregate.\n def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> Optional[ClarificationQuestion]:\n for q in clarification_session.questions:\n if q.question_id == question_id:\n return q\n return None\n\n # [/DEF:_find_question:Function]\n\n\n# [/DEF:ClarificationEngine:Class]\n\n# [/DEF:ClarificationEngine:Module]\n" + "body": "# [DEF:ClarificationEngine:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: dataset_review, clarification, question_payload, answer_persistence, readiness, findings\n# @PURPOSE: Manage one-question-at-a-time clarification state, deterministic answer persistence, and readiness/finding updates.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: DEPENDS_ON -> [ClarificationSession]\n# @RELATION: DEPENDS_ON -> [ClarificationQuestion]\n# @RELATION: DEPENDS_ON -> [ClarificationAnswer]\n# @RELATION: DEPENDS_ON -> [ValidationFinding]\n# @RELATION: DISPATCHES -> [ClarificationHelpers:Module]\n# @PRE: Target session contains a persisted clarification aggregate in the current ownership scope.\n# @POST: Active clarification payload exposes one highest-priority unresolved question, and each recorded answer is persisted before pointer/readiness mutation.\n# @SIDE_EFFECT: Persists clarification answers, question/session states, and related readiness/finding changes.\n# @DATA_CONTRACT: Input[DatasetReviewSession|ClarificationAnswerCommand] -> Output[ClarificationStateResult]\n# @INVARIANT: Only one active clarification question may exist at a time; skipped and expert-review items remain unresolved and visible.\n# @RATIONALE: Original 635-line file exceeded INV_7 (400-line module limit). Extracted pure helpers into _helpers sub-module.\n# @REJECTED: Keeping all clarification logic in one file because it exceeded the fractal limit.\n\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass, field\nfrom datetime import datetime\nfrom typing import List, Optional\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n AnswerKind,\n ClarificationAnswer,\n ClarificationQuestion,\n ClarificationSession,\n ClarificationStatus,\n DatasetReviewSession,\n QuestionState,\n ReadinessState,\n RecommendedAction,\n SessionPhase,\n ValidationFinding,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionRepository,\n)\nfrom src.services.dataset_review.clarification_pkg._helpers import (\n select_next_open_question,\n count_resolved_questions,\n count_remaining_questions,\n normalize_answer_value,\n build_impact_summary,\n upsert_clarification_finding,\n derive_readiness_state,\n derive_recommended_action,\n)\n\nlog = MarkerLogger(\"ClarificationEngine\")\n\n\n# [DEF:ClarificationQuestionPayload:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Typed active-question payload returned to the API layer.\n@dataclass\nclass ClarificationQuestionPayload:\n question_id: str\n clarification_session_id: str\n topic_ref: str\n question_text: str\n why_it_matters: str\n current_guess: Optional[str]\n priority: int\n state: QuestionState\n options: list[dict[str, object]] = field(default_factory=list)\n\n\n# [/DEF:ClarificationQuestionPayload:Class]\n\n\n# [DEF:ClarificationStateResult:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Clarification state result carrying the current session, active payload, and changed findings.\n@dataclass\nclass ClarificationStateResult:\n clarification_session: ClarificationSession\n current_question: Optional[ClarificationQuestionPayload]\n session: DatasetReviewSession\n changed_findings: List[ValidationFinding] = field(default_factory=list)\n\n\n# [/DEF:ClarificationStateResult:Class]\n\n\n# [DEF:ClarificationAnswerCommand:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Typed answer command for clarification state mutation.\n@dataclass\nclass ClarificationAnswerCommand:\n session: DatasetReviewSession\n question_id: str\n answer_kind: AnswerKind\n answer_value: Optional[str]\n user: User\n\n\n# [/DEF:ClarificationAnswerCommand:Class]\n\n\n# [DEF:ClarificationEngine:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Provide deterministic one-question-at-a-time clarification selection and answer persistence.\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: CALLS -> [ClarificationHelpers:Module]\n# @PRE: Repository is bound to the current request transaction scope.\n# @POST: Returned clarification state is persistence-backed and aligned with session readiness/recommended action.\n# @SIDE_EFFECT: Mutates clarification answers, session flags, and related clarification findings.\nclass ClarificationEngine:\n # [DEF:ClarificationEngine_init:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind repository dependency for clarification persistence operations.\n def __init__(self, repository: DatasetReviewSessionRepository) -> None:\n self.repository = repository\n\n # [/DEF:ClarificationEngine_init:Function]\n\n # [DEF:build_question_payload:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Return the one active highest-priority clarification question payload.\n # @PRE: Session contains unresolved clarification state or a resumable clarification session.\n # @POST: Returns exactly one active/open question payload or None when no unresolved question remains.\n # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence.\n def build_question_payload(\n self, session: DatasetReviewSession,\n ) -> Optional[ClarificationQuestionPayload]:\n with belief_scope(\"ClarificationEngine.build_question_payload\"):\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n log.reason(\"No clarification session found\", payload={\"session_id\": session.session_id})\n return None\n\n active_questions = [\n q for q in clarification_session.questions if q.state == QuestionState.OPEN\n ]\n active_questions.sort(key=lambda item: (-int(item.priority), item.created_at, item.question_id))\n\n if not active_questions:\n clarification_session.current_question_id = None\n clarification_session.status = ClarificationStatus.COMPLETED\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n if session.current_phase == SessionPhase.CLARIFICATION:\n session.current_phase = SessionPhase.REVIEW\n self.repository.db.commit()\n log.reflect(\"No unresolved clarification question remains\", payload={\"session_id\": session.session_id})\n return None\n\n selected_question = active_questions[0]\n clarification_session.current_question_id = selected_question.question_id\n clarification_session.status = ClarificationStatus.ACTIVE\n session.readiness_state = ReadinessState.CLARIFICATION_ACTIVE\n session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION\n session.current_phase = SessionPhase.CLARIFICATION\n\n log.reason(\"Selected active clarification question\", payload={\"session_id\": session.session_id, \"question_id\": selected_question.question_id, \"priority\": selected_question.priority})\n self.repository.db.commit()\n\n payload = ClarificationQuestionPayload(\n question_id=selected_question.question_id,\n clarification_session_id=selected_question.clarification_session_id,\n topic_ref=selected_question.topic_ref,\n question_text=selected_question.question_text,\n why_it_matters=selected_question.why_it_matters,\n current_guess=selected_question.current_guess,\n priority=selected_question.priority,\n state=selected_question.state,\n options=[\n {\"option_id\": o.option_id, \"question_id\": o.question_id, \"label\": o.label, \"value\": o.value, \"is_recommended\": o.is_recommended, \"display_order\": o.display_order}\n for o in sorted(selected_question.options, key=lambda item: (item.display_order, item.label, item.option_id))\n ],\n )\n log.reflect(\"Clarification payload built\", payload={\"session_id\": session.session_id, \"question_id\": payload.question_id, \"option_count\": len(payload.options)})\n return payload\n\n # [/DEF:build_question_payload:Function]\n\n # [DEF:record_answer:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation.\n # @PRE: Target question belongs to the session's active clarification session and is still open.\n # @POST: Answer row is persisted before current-question pointer advances.\n # @SIDE_EFFECT: Inserts answer row, mutates question/session states, updates clarification findings, and commits.\n def record_answer(self, command: ClarificationAnswerCommand) -> ClarificationStateResult:\n with belief_scope(\"ClarificationEngine.record_answer\"):\n session = command.session\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n log.explore(\"Cannot record clarification answer because no clarification session exists\", payload={\"session_id\": session.session_id}, error=\"No clarification session found\")\n raise ValueError(\"Clarification session not found\")\n\n question = self._find_question(clarification_session, command.question_id)\n if question is None:\n log.explore(\"Cannot record clarification answer for foreign or missing question\", payload={\"session_id\": session.session_id, \"question_id\": command.question_id}, error=\"Question not found in clarification session\")\n raise ValueError(\"Clarification question not found\")\n\n if question.answer is not None:\n log.explore(\"Rejected duplicate clarification answer submission\", payload={\"session_id\": session.session_id, \"question_id\": command.question_id}, error=\"Question already answered\")\n raise ValueError(\"Clarification question already answered\")\n\n if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id:\n log.explore(\"Rejected answer for non-active clarification question\", payload={\"session_id\": session.session_id, \"question_id\": question.question_id, \"current_question_id\": clarification_session.current_question_id}, error=\"Only active question can be answered\")\n raise ValueError(\"Only the active clarification question can be answered\")\n\n normalized_answer_value = normalize_answer_value(command.answer_kind, command.answer_value, question)\n\n log.reason(\"Persisting clarification answer before state advancement\", payload={\"session_id\": session.session_id, \"question_id\": question.question_id, \"answer_kind\": command.answer_kind.value})\n persisted_answer = ClarificationAnswer(\n question_id=question.question_id,\n answer_kind=command.answer_kind,\n answer_value=normalized_answer_value,\n answered_by_user_id=command.user.id,\n impact_summary=build_impact_summary(question, command.answer_kind, normalized_answer_value),\n )\n self.repository.db.add(persisted_answer)\n self.repository.db.flush()\n\n changed_finding = upsert_clarification_finding(\n session=session, question=question, answer_kind=command.answer_kind,\n answer_value=normalized_answer_value, db_session=self.repository.db,\n )\n\n if command.answer_kind == AnswerKind.SELECTED:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.CUSTOM:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.SKIPPED:\n question.state = QuestionState.SKIPPED\n elif command.answer_kind == AnswerKind.EXPERT_REVIEW:\n question.state = QuestionState.EXPERT_REVIEW\n\n question.updated_at = datetime.utcnow()\n self.repository.db.flush()\n\n clarification_session.resolved_count = count_resolved_questions(clarification_session)\n clarification_session.remaining_count = count_remaining_questions(clarification_session)\n clarification_session.summary_delta = self.summarize_progress(clarification_session)\n clarification_session.updated_at = datetime.utcnow()\n\n next_question = select_next_open_question(clarification_session)\n clarification_session.current_question_id = next_question.question_id if next_question else None\n clarification_session.status = ClarificationStatus.ACTIVE if next_question else ClarificationStatus.COMPLETED\n if clarification_session.status == ClarificationStatus.COMPLETED:\n clarification_session.completed_at = datetime.utcnow()\n\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n session.current_phase = SessionPhase.CLARIFICATION if clarification_session.current_question_id else SessionPhase.REVIEW\n self.repository.bump_session_version(session)\n\n self.repository.db.commit()\n self.repository.db.refresh(session)\n\n log.reflect(\"Clarification answer recorded and session advanced\", payload={\"session_id\": session.session_id, \"question_id\": question.question_id, \"next_question_id\": clarification_session.current_question_id, \"readiness_state\": session.readiness_state.value, \"remaining_count\": clarification_session.remaining_count})\n\n return ClarificationStateResult(\n clarification_session=clarification_session,\n current_question=self.build_question_payload(session),\n session=session,\n changed_findings=[changed_finding] if changed_finding else [],\n )\n\n # [/DEF:record_answer:Function]\n\n # [DEF:summarize_progress:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Produce a compact progress summary for pause/resume and completion UX.\n def summarize_progress(self, clarification_session: ClarificationSession) -> str:\n resolved = count_resolved_questions(clarification_session)\n remaining = count_remaining_questions(clarification_session)\n return f\"{resolved} resolved, {remaining} unresolved\"\n\n # [/DEF:summarize_progress:Function]\n\n # [DEF:_get_latest_clarification_session:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Select the latest clarification session for the current dataset review aggregate.\n def _get_latest_clarification_session(self, session: DatasetReviewSession) -> Optional[ClarificationSession]:\n if not session.clarification_sessions:\n return None\n ordered = sorted(session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True)\n return ordered[0]\n\n # [/DEF:_get_latest_clarification_session:Function]\n\n # [DEF:_find_question:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Resolve a clarification question from the active clarification aggregate.\n def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> Optional[ClarificationQuestion]:\n for q in clarification_session.questions:\n if q.question_id == question_id:\n return q\n return None\n\n # [/DEF:_find_question:Function]\n\n\n# [/DEF:ClarificationEngine:Class]\n\n# [/DEF:ClarificationEngine:Module]\n" }, { "contract_id": "ClarificationQuestionPayload", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 56, - "end_line": 72, + "start_line": 59, + "end_line": 75, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -80751,8 +81760,8 @@ "contract_id": "ClarificationStateResult", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 75, - "end_line": 86, + "start_line": 78, + "end_line": 89, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -80768,8 +81777,8 @@ "contract_id": "ClarificationAnswerCommand", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 89, - "end_line": 101, + "start_line": 92, + "end_line": 104, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -80785,8 +81794,8 @@ "contract_id": "ClarificationEngine_init", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 113, - "end_line": 119, + "start_line": 116, + "end_line": 122, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -80802,8 +81811,8 @@ "contract_id": "build_question_payload", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 121, - "end_line": 179, + "start_line": 124, + "end_line": 182, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -80826,14 +81835,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:build_question_payload:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Return the one active highest-priority clarification question payload.\n # @PRE: Session contains unresolved clarification state or a resumable clarification session.\n # @POST: Returns exactly one active/open question payload or None when no unresolved question remains.\n # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence.\n def build_question_payload(\n self, session: DatasetReviewSession,\n ) -> Optional[ClarificationQuestionPayload]:\n with belief_scope(\"ClarificationEngine.build_question_payload\"):\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n logger.reason(\"No clarification session found\", extra={\"session_id\": session.session_id})\n return None\n\n active_questions = [\n q for q in clarification_session.questions if q.state == QuestionState.OPEN\n ]\n active_questions.sort(key=lambda item: (-int(item.priority), item.created_at, item.question_id))\n\n if not active_questions:\n clarification_session.current_question_id = None\n clarification_session.status = ClarificationStatus.COMPLETED\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n if session.current_phase == SessionPhase.CLARIFICATION:\n session.current_phase = SessionPhase.REVIEW\n self.repository.db.commit()\n logger.reflect(\"No unresolved clarification question remains\", extra={\"session_id\": session.session_id})\n return None\n\n selected_question = active_questions[0]\n clarification_session.current_question_id = selected_question.question_id\n clarification_session.status = ClarificationStatus.ACTIVE\n session.readiness_state = ReadinessState.CLARIFICATION_ACTIVE\n session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION\n session.current_phase = SessionPhase.CLARIFICATION\n\n logger.reason(\"Selected active clarification question\", extra={\"session_id\": session.session_id, \"question_id\": selected_question.question_id, \"priority\": selected_question.priority})\n self.repository.db.commit()\n\n payload = ClarificationQuestionPayload(\n question_id=selected_question.question_id,\n clarification_session_id=selected_question.clarification_session_id,\n topic_ref=selected_question.topic_ref,\n question_text=selected_question.question_text,\n why_it_matters=selected_question.why_it_matters,\n current_guess=selected_question.current_guess,\n priority=selected_question.priority,\n state=selected_question.state,\n options=[\n {\"option_id\": o.option_id, \"question_id\": o.question_id, \"label\": o.label, \"value\": o.value, \"is_recommended\": o.is_recommended, \"display_order\": o.display_order}\n for o in sorted(selected_question.options, key=lambda item: (item.display_order, item.label, item.option_id))\n ],\n )\n logger.reflect(\"Clarification payload built\", extra={\"session_id\": session.session_id, \"question_id\": payload.question_id, \"option_count\": len(payload.options)})\n return payload\n\n # [/DEF:build_question_payload:Function]\n" + "body": " # [DEF:build_question_payload:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Return the one active highest-priority clarification question payload.\n # @PRE: Session contains unresolved clarification state or a resumable clarification session.\n # @POST: Returns exactly one active/open question payload or None when no unresolved question remains.\n # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence.\n def build_question_payload(\n self, session: DatasetReviewSession,\n ) -> Optional[ClarificationQuestionPayload]:\n with belief_scope(\"ClarificationEngine.build_question_payload\"):\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n log.reason(\"No clarification session found\", payload={\"session_id\": session.session_id})\n return None\n\n active_questions = [\n q for q in clarification_session.questions if q.state == QuestionState.OPEN\n ]\n active_questions.sort(key=lambda item: (-int(item.priority), item.created_at, item.question_id))\n\n if not active_questions:\n clarification_session.current_question_id = None\n clarification_session.status = ClarificationStatus.COMPLETED\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n if session.current_phase == SessionPhase.CLARIFICATION:\n session.current_phase = SessionPhase.REVIEW\n self.repository.db.commit()\n log.reflect(\"No unresolved clarification question remains\", payload={\"session_id\": session.session_id})\n return None\n\n selected_question = active_questions[0]\n clarification_session.current_question_id = selected_question.question_id\n clarification_session.status = ClarificationStatus.ACTIVE\n session.readiness_state = ReadinessState.CLARIFICATION_ACTIVE\n session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION\n session.current_phase = SessionPhase.CLARIFICATION\n\n log.reason(\"Selected active clarification question\", payload={\"session_id\": session.session_id, \"question_id\": selected_question.question_id, \"priority\": selected_question.priority})\n self.repository.db.commit()\n\n payload = ClarificationQuestionPayload(\n question_id=selected_question.question_id,\n clarification_session_id=selected_question.clarification_session_id,\n topic_ref=selected_question.topic_ref,\n question_text=selected_question.question_text,\n why_it_matters=selected_question.why_it_matters,\n current_guess=selected_question.current_guess,\n priority=selected_question.priority,\n state=selected_question.state,\n options=[\n {\"option_id\": o.option_id, \"question_id\": o.question_id, \"label\": o.label, \"value\": o.value, \"is_recommended\": o.is_recommended, \"display_order\": o.display_order}\n for o in sorted(selected_question.options, key=lambda item: (item.display_order, item.label, item.option_id))\n ],\n )\n log.reflect(\"Clarification payload built\", payload={\"session_id\": session.session_id, \"question_id\": payload.question_id, \"option_count\": len(payload.options)})\n return payload\n\n # [/DEF:build_question_payload:Function]\n" }, { "contract_id": "record_answer", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 181, - "end_line": 266, + "start_line": 184, + "end_line": 269, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -80856,14 +81865,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:record_answer:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation.\n # @PRE: Target question belongs to the session's active clarification session and is still open.\n # @POST: Answer row is persisted before current-question pointer advances.\n # @SIDE_EFFECT: Inserts answer row, mutates question/session states, updates clarification findings, and commits.\n def record_answer(self, command: ClarificationAnswerCommand) -> ClarificationStateResult:\n with belief_scope(\"ClarificationEngine.record_answer\"):\n session = command.session\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n logger.explore(\"Cannot record clarification answer because no clarification session exists\", extra={\"session_id\": session.session_id})\n raise ValueError(\"Clarification session not found\")\n\n question = self._find_question(clarification_session, command.question_id)\n if question is None:\n logger.explore(\"Cannot record clarification answer for foreign or missing question\", extra={\"session_id\": session.session_id, \"question_id\": command.question_id})\n raise ValueError(\"Clarification question not found\")\n\n if question.answer is not None:\n logger.explore(\"Rejected duplicate clarification answer submission\", extra={\"session_id\": session.session_id, \"question_id\": command.question_id})\n raise ValueError(\"Clarification question already answered\")\n\n if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id:\n logger.explore(\"Rejected answer for non-active clarification question\", extra={\"session_id\": session.session_id, \"question_id\": question.question_id, \"current_question_id\": clarification_session.current_question_id})\n raise ValueError(\"Only the active clarification question can be answered\")\n\n normalized_answer_value = normalize_answer_value(command.answer_kind, command.answer_value, question)\n\n logger.reason(\"Persisting clarification answer before state advancement\", extra={\"session_id\": session.session_id, \"question_id\": question.question_id, \"answer_kind\": command.answer_kind.value})\n persisted_answer = ClarificationAnswer(\n question_id=question.question_id,\n answer_kind=command.answer_kind,\n answer_value=normalized_answer_value,\n answered_by_user_id=command.user.id,\n impact_summary=build_impact_summary(question, command.answer_kind, normalized_answer_value),\n )\n self.repository.db.add(persisted_answer)\n self.repository.db.flush()\n\n changed_finding = upsert_clarification_finding(\n session=session, question=question, answer_kind=command.answer_kind,\n answer_value=normalized_answer_value, db_session=self.repository.db,\n )\n\n if command.answer_kind == AnswerKind.SELECTED:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.CUSTOM:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.SKIPPED:\n question.state = QuestionState.SKIPPED\n elif command.answer_kind == AnswerKind.EXPERT_REVIEW:\n question.state = QuestionState.EXPERT_REVIEW\n\n question.updated_at = datetime.utcnow()\n self.repository.db.flush()\n\n clarification_session.resolved_count = count_resolved_questions(clarification_session)\n clarification_session.remaining_count = count_remaining_questions(clarification_session)\n clarification_session.summary_delta = self.summarize_progress(clarification_session)\n clarification_session.updated_at = datetime.utcnow()\n\n next_question = select_next_open_question(clarification_session)\n clarification_session.current_question_id = next_question.question_id if next_question else None\n clarification_session.status = ClarificationStatus.ACTIVE if next_question else ClarificationStatus.COMPLETED\n if clarification_session.status == ClarificationStatus.COMPLETED:\n clarification_session.completed_at = datetime.utcnow()\n\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n session.current_phase = SessionPhase.CLARIFICATION if clarification_session.current_question_id else SessionPhase.REVIEW\n self.repository.bump_session_version(session)\n\n self.repository.db.commit()\n self.repository.db.refresh(session)\n\n logger.reflect(\"Clarification answer recorded and session advanced\", extra={\"session_id\": session.session_id, \"question_id\": question.question_id, \"next_question_id\": clarification_session.current_question_id, \"readiness_state\": session.readiness_state.value, \"remaining_count\": clarification_session.remaining_count})\n\n return ClarificationStateResult(\n clarification_session=clarification_session,\n current_question=self.build_question_payload(session),\n session=session,\n changed_findings=[changed_finding] if changed_finding else [],\n )\n\n # [/DEF:record_answer:Function]\n" + "body": " # [DEF:record_answer:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation.\n # @PRE: Target question belongs to the session's active clarification session and is still open.\n # @POST: Answer row is persisted before current-question pointer advances.\n # @SIDE_EFFECT: Inserts answer row, mutates question/session states, updates clarification findings, and commits.\n def record_answer(self, command: ClarificationAnswerCommand) -> ClarificationStateResult:\n with belief_scope(\"ClarificationEngine.record_answer\"):\n session = command.session\n clarification_session = self._get_latest_clarification_session(session)\n if clarification_session is None:\n log.explore(\"Cannot record clarification answer because no clarification session exists\", payload={\"session_id\": session.session_id}, error=\"No clarification session found\")\n raise ValueError(\"Clarification session not found\")\n\n question = self._find_question(clarification_session, command.question_id)\n if question is None:\n log.explore(\"Cannot record clarification answer for foreign or missing question\", payload={\"session_id\": session.session_id, \"question_id\": command.question_id}, error=\"Question not found in clarification session\")\n raise ValueError(\"Clarification question not found\")\n\n if question.answer is not None:\n log.explore(\"Rejected duplicate clarification answer submission\", payload={\"session_id\": session.session_id, \"question_id\": command.question_id}, error=\"Question already answered\")\n raise ValueError(\"Clarification question already answered\")\n\n if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id:\n log.explore(\"Rejected answer for non-active clarification question\", payload={\"session_id\": session.session_id, \"question_id\": question.question_id, \"current_question_id\": clarification_session.current_question_id}, error=\"Only active question can be answered\")\n raise ValueError(\"Only the active clarification question can be answered\")\n\n normalized_answer_value = normalize_answer_value(command.answer_kind, command.answer_value, question)\n\n log.reason(\"Persisting clarification answer before state advancement\", payload={\"session_id\": session.session_id, \"question_id\": question.question_id, \"answer_kind\": command.answer_kind.value})\n persisted_answer = ClarificationAnswer(\n question_id=question.question_id,\n answer_kind=command.answer_kind,\n answer_value=normalized_answer_value,\n answered_by_user_id=command.user.id,\n impact_summary=build_impact_summary(question, command.answer_kind, normalized_answer_value),\n )\n self.repository.db.add(persisted_answer)\n self.repository.db.flush()\n\n changed_finding = upsert_clarification_finding(\n session=session, question=question, answer_kind=command.answer_kind,\n answer_value=normalized_answer_value, db_session=self.repository.db,\n )\n\n if command.answer_kind == AnswerKind.SELECTED:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.CUSTOM:\n question.state = QuestionState.ANSWERED\n elif command.answer_kind == AnswerKind.SKIPPED:\n question.state = QuestionState.SKIPPED\n elif command.answer_kind == AnswerKind.EXPERT_REVIEW:\n question.state = QuestionState.EXPERT_REVIEW\n\n question.updated_at = datetime.utcnow()\n self.repository.db.flush()\n\n clarification_session.resolved_count = count_resolved_questions(clarification_session)\n clarification_session.remaining_count = count_remaining_questions(clarification_session)\n clarification_session.summary_delta = self.summarize_progress(clarification_session)\n clarification_session.updated_at = datetime.utcnow()\n\n next_question = select_next_open_question(clarification_session)\n clarification_session.current_question_id = next_question.question_id if next_question else None\n clarification_session.status = ClarificationStatus.ACTIVE if next_question else ClarificationStatus.COMPLETED\n if clarification_session.status == ClarificationStatus.COMPLETED:\n clarification_session.completed_at = datetime.utcnow()\n\n session.readiness_state = derive_readiness_state(session, clarification_session)\n session.recommended_action = derive_recommended_action(session, clarification_session)\n session.current_phase = SessionPhase.CLARIFICATION if clarification_session.current_question_id else SessionPhase.REVIEW\n self.repository.bump_session_version(session)\n\n self.repository.db.commit()\n self.repository.db.refresh(session)\n\n log.reflect(\"Clarification answer recorded and session advanced\", payload={\"session_id\": session.session_id, \"question_id\": question.question_id, \"next_question_id\": clarification_session.current_question_id, \"readiness_state\": session.readiness_state.value, \"remaining_count\": clarification_session.remaining_count})\n\n return ClarificationStateResult(\n clarification_session=clarification_session,\n current_question=self.build_question_payload(session),\n session=session,\n changed_findings=[changed_finding] if changed_finding else [],\n )\n\n # [/DEF:record_answer:Function]\n" }, { "contract_id": "summarize_progress", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 268, - "end_line": 276, + "start_line": 271, + "end_line": 279, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -80879,8 +81888,8 @@ "contract_id": "_get_latest_clarification_session", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 278, - "end_line": 287, + "start_line": 281, + "end_line": 290, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -80896,8 +81905,8 @@ "contract_id": "_find_question", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/clarification_engine.py", - "start_line": 289, - "end_line": 298, + "start_line": 292, + "end_line": 301, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -81082,7 +82091,7 @@ "contract_type": "Module", "file_path": "backend/src/services/dataset_review/event_logger.py", "start_line": 1, - "end_line": 158, + "end_line": 152, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81161,14 +82170,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SessionEventLoggerModule:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: dataset_review, audit, session_events, persistence, observability\n# @PURPOSE: Persist explicit session mutation events for dataset-review audit trails without weakening ownership or approval invariants.\n# @LAYER: Domain\n# @RELATION: [DEPENDS_ON] ->[SessionEvent]\n# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession]\n# @PRE: Caller provides an owned session scope and an authenticated actor identifier for each persisted mutation event.\n# @POST: Every logged event is committed as an explicit, queryable audit record with deterministic event metadata.\n# @SIDE_EFFECT: Inserts persisted session event rows and emits runtime belief-state logs for audit-sensitive mutations.\n# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n\nfrom __future__ import annotations\n\n# [DEF:SessionEventLoggerImports:Block]\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, Optional\n\nfrom sqlalchemy.orm import Session\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.dataset_review import DatasetReviewSession, SessionEvent\n# [/DEF:SessionEventLoggerImports:Block]\n\n\n# [DEF:SessionEventPayload:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Typed input contract for one persisted dataset-review session audit event.\n@dataclass(frozen=True)\nclass SessionEventPayload:\n session_id: str\n actor_user_id: str\n event_type: str\n event_summary: str\n current_phase: Optional[str] = None\n readiness_state: Optional[str] = None\n event_details: Dict[str, Any] = field(default_factory=dict)\n# [/DEF:SessionEventPayload:Class]\n\n\n# [DEF:SessionEventLogger:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist explicit dataset-review session audit events with meaningful runtime reasoning logs.\n# @RELATION: [DEPENDS_ON] ->[SessionEvent]\n# @RELATION: [DEPENDS_ON] ->[SessionEventPayload]\n# @PRE: The database session is live and payload identifiers are non-empty.\n# @POST: Returns the committed session event row with a stable identifier and stored detail payload.\n# @SIDE_EFFECT: Writes one audit row to persistence and emits logger.reason/logger.reflect traces.\n# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\nclass SessionEventLogger:\n # [DEF:SessionEventLogger_init:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger.\n def __init__(self, db: Session) -> None:\n self.db = db\n # [/DEF:SessionEventLogger_init:Function]\n\n # [DEF:log_event:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.\n # @RELATION: [DEPENDS_ON] ->[SessionEvent]\n # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.\n # @POST: Returns the committed SessionEvent record with normalized detail payload.\n # @SIDE_EFFECT: Inserts and commits one session_events row.\n # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n def log_event(self, payload: SessionEventPayload) -> SessionEvent:\n with belief_scope(\"SessionEventLogger.log_event\"):\n session_id = str(payload.session_id or \"\").strip()\n actor_user_id = str(payload.actor_user_id or \"\").strip()\n event_type = str(payload.event_type or \"\").strip()\n event_summary = str(payload.event_summary or \"\").strip()\n\n if not session_id:\n logger.explore(\"Session event logging rejected because session_id is empty\")\n raise ValueError(\"session_id must be non-empty\")\n if not actor_user_id:\n logger.explore(\n \"Session event logging rejected because actor_user_id is empty\",\n extra={\"session_id\": session_id},\n )\n raise ValueError(\"actor_user_id must be non-empty\")\n if not event_type:\n logger.explore(\n \"Session event logging rejected because event_type is empty\",\n extra={\"session_id\": session_id, \"actor_user_id\": actor_user_id},\n )\n raise ValueError(\"event_type must be non-empty\")\n if not event_summary:\n logger.explore(\n \"Session event logging rejected because event_summary is empty\",\n extra={\"session_id\": session_id, \"event_type\": event_type},\n )\n raise ValueError(\"event_summary must be non-empty\")\n\n normalized_details = dict(payload.event_details or {})\n logger.reason(\n \"Persisting explicit dataset-review session audit event\",\n extra={\n \"session_id\": session_id,\n \"actor_user_id\": actor_user_id,\n \"event_type\": event_type,\n \"current_phase\": payload.current_phase,\n \"readiness_state\": payload.readiness_state,\n },\n )\n\n event = SessionEvent(\n session_id=session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=payload.current_phase,\n readiness_state=payload.readiness_state,\n event_details=normalized_details,\n )\n self.db.add(event)\n self.db.commit()\n self.db.refresh(event)\n\n logger.reflect(\n \"Dataset-review session audit event persisted\",\n extra={\n \"session_id\": session_id,\n \"session_event_id\": event.session_event_id,\n \"event_type\": event.event_type,\n },\n )\n return event\n # [/DEF:log_event:Function]\n\n # [DEF:log_for_session:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root.\n # @RELATION: [CALLS] ->[SessionEventLogger.log_event]\n def log_for_session(\n self,\n session: DatasetReviewSession,\n *,\n actor_user_id: str,\n event_type: str,\n event_summary: str,\n event_details: Optional[Dict[str, Any]] = None,\n ) -> SessionEvent:\n return self.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=session.current_phase.value if session.current_phase else None,\n readiness_state=session.readiness_state.value if session.readiness_state else None,\n event_details=dict(event_details or {}),\n )\n )\n # [/DEF:log_for_session:Function]\n# [/DEF:SessionEventLogger:Class]\n\n# [/DEF:SessionEventLoggerModule:Module]\n" + "body": "# [DEF:SessionEventLoggerModule:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: dataset_review, audit, session_events, persistence, observability\n# @PURPOSE: Persist explicit session mutation events for dataset-review audit trails without weakening ownership or approval invariants.\n# @LAYER: Domain\n# @RELATION: [DEPENDS_ON] ->[SessionEvent]\n# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession]\n# @PRE: Caller provides an owned session scope and an authenticated actor identifier for each persisted mutation event.\n# @POST: Every logged event is committed as an explicit, queryable audit record with deterministic event metadata.\n# @SIDE_EFFECT: Inserts persisted session event rows and emits runtime belief-state logs for audit-sensitive mutations.\n# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n\nfrom __future__ import annotations\n\n# [DEF:SessionEventLoggerImports:Block]\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, Optional\n\nfrom sqlalchemy.orm import Session\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.dataset_review import DatasetReviewSession, SessionEvent\n# [/DEF:SessionEventLoggerImports:Block]\n\nlog = MarkerLogger(\"SessionEventLogger\")\n\n\n# [DEF:SessionEventPayload:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Typed input contract for one persisted dataset-review session audit event.\n@dataclass(frozen=True)\nclass SessionEventPayload:\n session_id: str\n actor_user_id: str\n event_type: str\n event_summary: str\n current_phase: Optional[str] = None\n readiness_state: Optional[str] = None\n event_details: Dict[str, Any] = field(default_factory=dict)\n# [/DEF:SessionEventPayload:Class]\n\n\n# [DEF:SessionEventLogger:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist explicit dataset-review session audit events with meaningful runtime reasoning logs.\n# @RELATION: [DEPENDS_ON] ->[SessionEvent]\n# @RELATION: [DEPENDS_ON] ->[SessionEventPayload]\n# @PRE: The database session is live and payload identifiers are non-empty.\n# @POST: Returns the committed session event row with a stable identifier and stored detail payload.\n# @SIDE_EFFECT: Writes one audit row to persistence and emits logger.reason/logger.reflect traces.\n# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\nclass SessionEventLogger:\n # [DEF:SessionEventLogger_init:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger.\n def __init__(self, db: Session) -> None:\n self.db = db\n # [/DEF:SessionEventLogger_init:Function]\n\n # [DEF:log_event:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.\n # @RELATION: [DEPENDS_ON] ->[SessionEvent]\n # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.\n # @POST: Returns the committed SessionEvent record with normalized detail payload.\n # @SIDE_EFFECT: Inserts and commits one session_events row.\n # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n def log_event(self, payload: SessionEventPayload) -> SessionEvent:\n with belief_scope(\"SessionEventLogger.log_event\"):\n session_id = str(payload.session_id or \"\").strip()\n actor_user_id = str(payload.actor_user_id or \"\").strip()\n event_type = str(payload.event_type or \"\").strip()\n event_summary = str(payload.event_summary or \"\").strip()\n\n if not session_id:\n log.explore(\"Session event logging rejected because session_id is empty\", error=\"session_id is empty\")\n raise ValueError(\"session_id must be non-empty\")\n if not actor_user_id:\n log.explore(\"Session event logging rejected because actor_user_id is empty\", error=\"actor_user_id is empty\", payload={\"session_id\": session_id})\n raise ValueError(\"actor_user_id must be non-empty\")\n if not event_type:\n log.explore(\"Session event logging rejected because event_type is empty\", error=\"event_type is empty\", payload={\"session_id\": session_id, \"actor_user_id\": actor_user_id})\n raise ValueError(\"event_type must be non-empty\")\n if not event_summary:\n log.explore(\"Session event logging rejected because event_summary is empty\", error=\"event_summary is empty\", payload={\"session_id\": session_id, \"event_type\": event_type})\n raise ValueError(\"event_summary must be non-empty\")\n\n normalized_details = dict(payload.event_details or {})\n log.reason(\n \"Persisting explicit dataset-review session audit event\",\n payload={\n \"session_id\": session_id,\n \"actor_user_id\": actor_user_id,\n \"event_type\": event_type,\n \"current_phase\": payload.current_phase,\n \"readiness_state\": payload.readiness_state,\n },\n )\n\n event = SessionEvent(\n session_id=session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=payload.current_phase,\n readiness_state=payload.readiness_state,\n event_details=normalized_details,\n )\n self.db.add(event)\n self.db.commit()\n self.db.refresh(event)\n\n log.reflect(\n \"Dataset-review session audit event persisted\",\n payload={\n \"session_id\": session_id,\n \"session_event_id\": event.session_event_id,\n \"event_type\": event.event_type,\n },\n )\n return event\n # [/DEF:log_event:Function]\n\n # [DEF:log_for_session:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root.\n # @RELATION: [CALLS] ->[SessionEventLogger.log_event]\n def log_for_session(\n self,\n session: DatasetReviewSession,\n *,\n actor_user_id: str,\n event_type: str,\n event_summary: str,\n event_details: Optional[Dict[str, Any]] = None,\n ) -> SessionEvent:\n return self.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=session.current_phase.value if session.current_phase else None,\n readiness_state=session.readiness_state.value if session.readiness_state else None,\n event_details=dict(event_details or {}),\n )\n )\n # [/DEF:log_for_session:Function]\n# [/DEF:SessionEventLogger:Class]\n\n# [/DEF:SessionEventLoggerModule:Module]\n" }, { "contract_id": "SessionEventLoggerImports", "contract_type": "Block", "file_path": "backend/src/services/dataset_review/event_logger.py", "start_line": 15, - "end_line": 23, + "end_line": 24, "tier": "TIER_1", "complexity": 1, "metadata": {}, @@ -81185,14 +82194,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SessionEventLoggerImports:Block]\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, Optional\n\nfrom sqlalchemy.orm import Session\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.dataset_review import DatasetReviewSession, SessionEvent\n# [/DEF:SessionEventLoggerImports:Block]\n" + "body": "# [DEF:SessionEventLoggerImports:Block]\nfrom dataclasses import dataclass, field\nfrom typing import Any, Dict, Optional\n\nfrom sqlalchemy.orm import Session\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.dataset_review import DatasetReviewSession, SessionEvent\n# [/DEF:SessionEventLoggerImports:Block]\n" }, { "contract_id": "SessionEventPayload", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/event_logger.py", - "start_line": 26, - "end_line": 38, + "start_line": 29, + "end_line": 41, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -81208,8 +82217,8 @@ "contract_id": "SessionEventLogger", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/event_logger.py", - "start_line": 41, - "end_line": 156, + "start_line": 44, + "end_line": 150, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81280,14 +82289,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SessionEventLogger:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist explicit dataset-review session audit events with meaningful runtime reasoning logs.\n# @RELATION: [DEPENDS_ON] ->[SessionEvent]\n# @RELATION: [DEPENDS_ON] ->[SessionEventPayload]\n# @PRE: The database session is live and payload identifiers are non-empty.\n# @POST: Returns the committed session event row with a stable identifier and stored detail payload.\n# @SIDE_EFFECT: Writes one audit row to persistence and emits logger.reason/logger.reflect traces.\n# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\nclass SessionEventLogger:\n # [DEF:SessionEventLogger_init:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger.\n def __init__(self, db: Session) -> None:\n self.db = db\n # [/DEF:SessionEventLogger_init:Function]\n\n # [DEF:log_event:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.\n # @RELATION: [DEPENDS_ON] ->[SessionEvent]\n # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.\n # @POST: Returns the committed SessionEvent record with normalized detail payload.\n # @SIDE_EFFECT: Inserts and commits one session_events row.\n # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n def log_event(self, payload: SessionEventPayload) -> SessionEvent:\n with belief_scope(\"SessionEventLogger.log_event\"):\n session_id = str(payload.session_id or \"\").strip()\n actor_user_id = str(payload.actor_user_id or \"\").strip()\n event_type = str(payload.event_type or \"\").strip()\n event_summary = str(payload.event_summary or \"\").strip()\n\n if not session_id:\n logger.explore(\"Session event logging rejected because session_id is empty\")\n raise ValueError(\"session_id must be non-empty\")\n if not actor_user_id:\n logger.explore(\n \"Session event logging rejected because actor_user_id is empty\",\n extra={\"session_id\": session_id},\n )\n raise ValueError(\"actor_user_id must be non-empty\")\n if not event_type:\n logger.explore(\n \"Session event logging rejected because event_type is empty\",\n extra={\"session_id\": session_id, \"actor_user_id\": actor_user_id},\n )\n raise ValueError(\"event_type must be non-empty\")\n if not event_summary:\n logger.explore(\n \"Session event logging rejected because event_summary is empty\",\n extra={\"session_id\": session_id, \"event_type\": event_type},\n )\n raise ValueError(\"event_summary must be non-empty\")\n\n normalized_details = dict(payload.event_details or {})\n logger.reason(\n \"Persisting explicit dataset-review session audit event\",\n extra={\n \"session_id\": session_id,\n \"actor_user_id\": actor_user_id,\n \"event_type\": event_type,\n \"current_phase\": payload.current_phase,\n \"readiness_state\": payload.readiness_state,\n },\n )\n\n event = SessionEvent(\n session_id=session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=payload.current_phase,\n readiness_state=payload.readiness_state,\n event_details=normalized_details,\n )\n self.db.add(event)\n self.db.commit()\n self.db.refresh(event)\n\n logger.reflect(\n \"Dataset-review session audit event persisted\",\n extra={\n \"session_id\": session_id,\n \"session_event_id\": event.session_event_id,\n \"event_type\": event.event_type,\n },\n )\n return event\n # [/DEF:log_event:Function]\n\n # [DEF:log_for_session:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root.\n # @RELATION: [CALLS] ->[SessionEventLogger.log_event]\n def log_for_session(\n self,\n session: DatasetReviewSession,\n *,\n actor_user_id: str,\n event_type: str,\n event_summary: str,\n event_details: Optional[Dict[str, Any]] = None,\n ) -> SessionEvent:\n return self.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=session.current_phase.value if session.current_phase else None,\n readiness_state=session.readiness_state.value if session.readiness_state else None,\n event_details=dict(event_details or {}),\n )\n )\n # [/DEF:log_for_session:Function]\n# [/DEF:SessionEventLogger:Class]\n" + "body": "# [DEF:SessionEventLogger:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist explicit dataset-review session audit events with meaningful runtime reasoning logs.\n# @RELATION: [DEPENDS_ON] ->[SessionEvent]\n# @RELATION: [DEPENDS_ON] ->[SessionEventPayload]\n# @PRE: The database session is live and payload identifiers are non-empty.\n# @POST: Returns the committed session event row with a stable identifier and stored detail payload.\n# @SIDE_EFFECT: Writes one audit row to persistence and emits logger.reason/logger.reflect traces.\n# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\nclass SessionEventLogger:\n # [DEF:SessionEventLogger_init:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger.\n def __init__(self, db: Session) -> None:\n self.db = db\n # [/DEF:SessionEventLogger_init:Function]\n\n # [DEF:log_event:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.\n # @RELATION: [DEPENDS_ON] ->[SessionEvent]\n # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.\n # @POST: Returns the committed SessionEvent record with normalized detail payload.\n # @SIDE_EFFECT: Inserts and commits one session_events row.\n # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n def log_event(self, payload: SessionEventPayload) -> SessionEvent:\n with belief_scope(\"SessionEventLogger.log_event\"):\n session_id = str(payload.session_id or \"\").strip()\n actor_user_id = str(payload.actor_user_id or \"\").strip()\n event_type = str(payload.event_type or \"\").strip()\n event_summary = str(payload.event_summary or \"\").strip()\n\n if not session_id:\n log.explore(\"Session event logging rejected because session_id is empty\", error=\"session_id is empty\")\n raise ValueError(\"session_id must be non-empty\")\n if not actor_user_id:\n log.explore(\"Session event logging rejected because actor_user_id is empty\", error=\"actor_user_id is empty\", payload={\"session_id\": session_id})\n raise ValueError(\"actor_user_id must be non-empty\")\n if not event_type:\n log.explore(\"Session event logging rejected because event_type is empty\", error=\"event_type is empty\", payload={\"session_id\": session_id, \"actor_user_id\": actor_user_id})\n raise ValueError(\"event_type must be non-empty\")\n if not event_summary:\n log.explore(\"Session event logging rejected because event_summary is empty\", error=\"event_summary is empty\", payload={\"session_id\": session_id, \"event_type\": event_type})\n raise ValueError(\"event_summary must be non-empty\")\n\n normalized_details = dict(payload.event_details or {})\n log.reason(\n \"Persisting explicit dataset-review session audit event\",\n payload={\n \"session_id\": session_id,\n \"actor_user_id\": actor_user_id,\n \"event_type\": event_type,\n \"current_phase\": payload.current_phase,\n \"readiness_state\": payload.readiness_state,\n },\n )\n\n event = SessionEvent(\n session_id=session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=payload.current_phase,\n readiness_state=payload.readiness_state,\n event_details=normalized_details,\n )\n self.db.add(event)\n self.db.commit()\n self.db.refresh(event)\n\n log.reflect(\n \"Dataset-review session audit event persisted\",\n payload={\n \"session_id\": session_id,\n \"session_event_id\": event.session_event_id,\n \"event_type\": event.event_type,\n },\n )\n return event\n # [/DEF:log_event:Function]\n\n # [DEF:log_for_session:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root.\n # @RELATION: [CALLS] ->[SessionEventLogger.log_event]\n def log_for_session(\n self,\n session: DatasetReviewSession,\n *,\n actor_user_id: str,\n event_type: str,\n event_summary: str,\n event_details: Optional[Dict[str, Any]] = None,\n ) -> SessionEvent:\n return self.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=session.current_phase.value if session.current_phase else None,\n readiness_state=session.readiness_state.value if session.readiness_state else None,\n event_details=dict(event_details or {}),\n )\n )\n # [/DEF:log_for_session:Function]\n# [/DEF:SessionEventLogger:Class]\n" }, { "contract_id": "SessionEventLogger_init", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/event_logger.py", - "start_line": 51, - "end_line": 56, + "start_line": 54, + "end_line": 59, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -81303,8 +82312,8 @@ "contract_id": "log_event", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/event_logger.py", - "start_line": 58, - "end_line": 129, + "start_line": 61, + "end_line": 123, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81352,14 +82361,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:log_event:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.\n # @RELATION: [DEPENDS_ON] ->[SessionEvent]\n # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.\n # @POST: Returns the committed SessionEvent record with normalized detail payload.\n # @SIDE_EFFECT: Inserts and commits one session_events row.\n # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n def log_event(self, payload: SessionEventPayload) -> SessionEvent:\n with belief_scope(\"SessionEventLogger.log_event\"):\n session_id = str(payload.session_id or \"\").strip()\n actor_user_id = str(payload.actor_user_id or \"\").strip()\n event_type = str(payload.event_type or \"\").strip()\n event_summary = str(payload.event_summary or \"\").strip()\n\n if not session_id:\n logger.explore(\"Session event logging rejected because session_id is empty\")\n raise ValueError(\"session_id must be non-empty\")\n if not actor_user_id:\n logger.explore(\n \"Session event logging rejected because actor_user_id is empty\",\n extra={\"session_id\": session_id},\n )\n raise ValueError(\"actor_user_id must be non-empty\")\n if not event_type:\n logger.explore(\n \"Session event logging rejected because event_type is empty\",\n extra={\"session_id\": session_id, \"actor_user_id\": actor_user_id},\n )\n raise ValueError(\"event_type must be non-empty\")\n if not event_summary:\n logger.explore(\n \"Session event logging rejected because event_summary is empty\",\n extra={\"session_id\": session_id, \"event_type\": event_type},\n )\n raise ValueError(\"event_summary must be non-empty\")\n\n normalized_details = dict(payload.event_details or {})\n logger.reason(\n \"Persisting explicit dataset-review session audit event\",\n extra={\n \"session_id\": session_id,\n \"actor_user_id\": actor_user_id,\n \"event_type\": event_type,\n \"current_phase\": payload.current_phase,\n \"readiness_state\": payload.readiness_state,\n },\n )\n\n event = SessionEvent(\n session_id=session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=payload.current_phase,\n readiness_state=payload.readiness_state,\n event_details=normalized_details,\n )\n self.db.add(event)\n self.db.commit()\n self.db.refresh(event)\n\n logger.reflect(\n \"Dataset-review session audit event persisted\",\n extra={\n \"session_id\": session_id,\n \"session_event_id\": event.session_event_id,\n \"event_type\": event.event_type,\n },\n )\n return event\n # [/DEF:log_event:Function]\n" + "body": " # [DEF:log_event:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation.\n # @RELATION: [DEPENDS_ON] ->[SessionEvent]\n # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty.\n # @POST: Returns the committed SessionEvent record with normalized detail payload.\n # @SIDE_EFFECT: Inserts and commits one session_events row.\n # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent]\n def log_event(self, payload: SessionEventPayload) -> SessionEvent:\n with belief_scope(\"SessionEventLogger.log_event\"):\n session_id = str(payload.session_id or \"\").strip()\n actor_user_id = str(payload.actor_user_id or \"\").strip()\n event_type = str(payload.event_type or \"\").strip()\n event_summary = str(payload.event_summary or \"\").strip()\n\n if not session_id:\n log.explore(\"Session event logging rejected because session_id is empty\", error=\"session_id is empty\")\n raise ValueError(\"session_id must be non-empty\")\n if not actor_user_id:\n log.explore(\"Session event logging rejected because actor_user_id is empty\", error=\"actor_user_id is empty\", payload={\"session_id\": session_id})\n raise ValueError(\"actor_user_id must be non-empty\")\n if not event_type:\n log.explore(\"Session event logging rejected because event_type is empty\", error=\"event_type is empty\", payload={\"session_id\": session_id, \"actor_user_id\": actor_user_id})\n raise ValueError(\"event_type must be non-empty\")\n if not event_summary:\n log.explore(\"Session event logging rejected because event_summary is empty\", error=\"event_summary is empty\", payload={\"session_id\": session_id, \"event_type\": event_type})\n raise ValueError(\"event_summary must be non-empty\")\n\n normalized_details = dict(payload.event_details or {})\n log.reason(\n \"Persisting explicit dataset-review session audit event\",\n payload={\n \"session_id\": session_id,\n \"actor_user_id\": actor_user_id,\n \"event_type\": event_type,\n \"current_phase\": payload.current_phase,\n \"readiness_state\": payload.readiness_state,\n },\n )\n\n event = SessionEvent(\n session_id=session_id,\n actor_user_id=actor_user_id,\n event_type=event_type,\n event_summary=event_summary,\n current_phase=payload.current_phase,\n readiness_state=payload.readiness_state,\n event_details=normalized_details,\n )\n self.db.add(event)\n self.db.commit()\n self.db.refresh(event)\n\n log.reflect(\n \"Dataset-review session audit event persisted\",\n payload={\n \"session_id\": session_id,\n \"session_event_id\": event.session_event_id,\n \"event_type\": event.event_type,\n },\n )\n return event\n # [/DEF:log_event:Function]\n" }, { "contract_id": "log_for_session", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/event_logger.py", - "start_line": 131, - "end_line": 155, + "start_line": 125, + "end_line": 149, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -81410,7 +82419,7 @@ "contract_type": "Module", "file_path": "backend/src/services/dataset_review/orchestrator.py", "start_line": 1, - "end_line": 612, + "end_line": 613, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -81478,14 +82487,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:DatasetReviewOrchestrator:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: dataset_review, orchestration, session_lifecycle, intake, recovery\n# @PURPOSE: Coordinate dataset review session startup and lifecycle-safe intake recovery for one authenticated user.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: DEPENDS_ON -> [SemanticSourceResolver]\n# @RELATION: DEPENDS_ON -> [SupersetContextExtractor]\n# @RELATION: DEPENDS_ON -> [SupersetCompilationAdapter]\n# @RELATION: DEPENDS_ON -> [TaskManager]\n# @RELATION: DISPATCHES -> [OrchestratorHelpers:Module]\n# @RELATION: DISPATCHES -> [OrchestratorCommands:Module]\n# @PRE: session mutations must execute inside a persisted session boundary scoped to one authenticated user.\n# @POST: state transitions are persisted atomically and emit observable progress for long-running steps.\n# @SIDE_EFFECT: creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts.\n# @DATA_CONTRACT: Input[SessionCommand] -> Output[DatasetReviewSession | CompiledPreview | DatasetRunContext]\n# @INVARIANT: Launch is blocked unless a current session has no open blocking findings, all launch-sensitive mappings are approved, and a non-stale Superset-generated compiled preview matches the current input fingerprint.\n# @RATIONALE: Original 1198-line monolith violated INV_7 (400-line module limit). Decomposed into commands and helpers sub-modules while preserving the orchestrator class as the single entry point.\n# @REJECTED: Keeping all orchestration logic in one file because it exceeded the fractal limit by 3x.\n\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass, field\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional, cast\n\nfrom src.core.config_manager import ConfigManager\nfrom src.core.logger import belief_scope, logger\nfrom src.core.task_manager import TaskManager\nfrom src.core.utils.superset_compilation_adapter import (\n PreviewCompilationPayload,\n SqlLabLaunchPayload,\n SupersetCompilationAdapter,\n)\nfrom src.core.utils.superset_context_extractor import (\n SupersetContextExtractor,\n SupersetParsedContext,\n)\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n ApprovalState,\n BusinessSummarySource,\n CompiledPreview,\n ConfidenceState,\n DatasetProfile,\n DatasetReviewSession,\n DatasetRunContext,\n ExecutionMapping,\n FilterConfidenceState,\n FilterRecoveryStatus,\n FilterSource,\n FindingArea,\n FindingSeverity,\n ImportedFilter,\n LaunchStatus,\n MappingMethod,\n MappingStatus,\n PreviewStatus,\n RecommendedAction,\n ReadinessState,\n ResolutionState,\n SessionPhase,\n SessionStatus,\n TemplateVariable,\n ValidationFinding,\n VariableKind,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionRepository,\n)\nfrom src.services.dataset_review.semantic_resolver import SemanticSourceResolver\nfrom src.services.dataset_review.event_logger import SessionEventPayload\nfrom src.services.dataset_review.orchestrator_pkg._commands import (\n StartSessionCommand,\n StartSessionResult,\n PreparePreviewCommand,\n PreparePreviewResult,\n LaunchDatasetCommand,\n LaunchDatasetResult,\n)\nfrom src.services.dataset_review.orchestrator_pkg._helpers import (\n parse_dataset_selection,\n build_initial_profile,\n build_partial_recovery_findings,\n build_execution_snapshot,\n build_launch_blockers,\n get_latest_preview,\n compute_preview_fingerprint,\n extract_effective_filter_value,\n)\n\nlogger = cast(Any, logger)\n\n\n# [DEF:DatasetReviewOrchestrator:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Coordinate safe session startup while preserving cross-user isolation and explicit partial recovery.\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: DEPENDS_ON -> [SupersetContextExtractor]\n# @RELATION: DEPENDS_ON -> [TaskManager]\n# @RELATION: DEPENDS_ON -> [ConfigManager]\n# @RELATION: DEPENDS_ON -> [SemanticSourceResolver]\n# @RELATION: CALLS -> [OrchestratorHelpers:Module]\n# @PRE: constructor dependencies are valid and tied to the current request/task scope.\n# @POST: orchestrator instance can execute session-scoped mutations for one authenticated user.\n# @SIDE_EFFECT: downstream operations may persist session/profile/finding state and enqueue background tasks.\n# @DATA_CONTRACT: Input[StartSessionCommand] -> Output[StartSessionResult]\n# @INVARIANT: session ownership is preserved on every mutation and recovery remains explicit when partial.\nclass DatasetReviewOrchestrator:\n # [DEF:DatasetReviewOrchestrator_init:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary.\n # @PRE: repository/config_manager are valid collaborators for the current request scope.\n # @POST: Instance holds collaborator references used by start/preview/launch orchestration methods.\n def __init__(\n self,\n repository: DatasetReviewSessionRepository,\n config_manager: ConfigManager,\n task_manager: Optional[TaskManager] = None,\n semantic_resolver: Optional[SemanticSourceResolver] = None,\n ) -> None:\n self.repository = repository\n self.config_manager = config_manager\n self.task_manager = task_manager\n self.semantic_resolver = semantic_resolver or SemanticSourceResolver()\n\n # [/DEF:DatasetReviewOrchestrator_init:Function]\n\n # [DEF:start_session:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery.\n # @RELATION: CALLS -> [SupersetContextExtractor.parse_superset_link]\n # @RELATION: CALLS -> [TaskManager.create_task]\n # @PRE: source input is non-empty and environment is accessible.\n # @POST: session exists in persisted storage with intake/recovery state and task linkage when async work is required.\n # @SIDE_EFFECT: persists session and may enqueue recovery task.\n # @DATA_CONTRACT: Input[StartSessionCommand] -> Output[StartSessionResult]\n # @INVARIANT: no cross-user session leakage occurs; session and follow-up task remain owned by the authenticated user.\n def start_session(self, command: StartSessionCommand) -> StartSessionResult:\n with belief_scope(\"DatasetReviewOrchestrator.start_session\"):\n normalized_source_kind = str(command.source_kind or \"\").strip()\n normalized_source_input = str(command.source_input or \"\").strip()\n normalized_environment_id = str(command.environment_id or \"\").strip()\n\n if not normalized_source_input:\n logger.explore(\"Blocked dataset review session start due to empty source input\")\n raise ValueError(\"source_input must be non-empty\")\n\n if normalized_source_kind not in {\"superset_link\", \"dataset_selection\"}:\n logger.explore(\"Blocked dataset review session start due to unsupported source kind\", extra={\"source_kind\": normalized_source_kind})\n raise ValueError(\"source_kind must be 'superset_link' or 'dataset_selection'\")\n\n environment = self.config_manager.get_environment(normalized_environment_id)\n if environment is None:\n logger.explore(\"Blocked dataset review session start because environment was not found\", extra={\"environment_id\": normalized_environment_id})\n raise ValueError(\"Environment not found\")\n\n logger.reason(\"Starting dataset review session\", extra={\"user_id\": command.user.id, \"environment_id\": normalized_environment_id, \"source_kind\": normalized_source_kind})\n\n parsed_context: Optional[SupersetParsedContext] = None\n findings: List[ValidationFinding] = []\n dataset_ref = normalized_source_input\n dataset_id: Optional[int] = None\n dashboard_id: Optional[int] = None\n readiness_state = ReadinessState.IMPORTING\n recommended_action = RecommendedAction.REVIEW_DOCUMENTATION\n current_phase = SessionPhase.RECOVERY\n\n if normalized_source_kind == \"superset_link\":\n extractor = SupersetContextExtractor(environment)\n parsed_context = extractor.parse_superset_link(normalized_source_input)\n dataset_ref = parsed_context.dataset_ref\n dataset_id = parsed_context.dataset_id\n dashboard_id = parsed_context.dashboard_id\n\n if parsed_context.partial_recovery:\n readiness_state = ReadinessState.RECOVERY_REQUIRED\n recommended_action = RecommendedAction.REVIEW_DOCUMENTATION\n findings.extend(build_partial_recovery_findings(parsed_context))\n else:\n readiness_state = ReadinessState.REVIEW_READY\n else:\n dataset_ref, dataset_id = parse_dataset_selection(normalized_source_input)\n readiness_state = ReadinessState.REVIEW_READY\n current_phase = SessionPhase.REVIEW\n\n session = DatasetReviewSession(\n user_id=command.user.id,\n environment_id=normalized_environment_id,\n source_kind=normalized_source_kind,\n source_input=normalized_source_input,\n dataset_ref=dataset_ref,\n dataset_id=dataset_id,\n dashboard_id=dashboard_id,\n readiness_state=readiness_state,\n recommended_action=recommended_action,\n status=SessionStatus.ACTIVE,\n current_phase=current_phase,\n )\n persisted_session = cast(Any, self.repository.create_session(session))\n\n recovered_filters: List[ImportedFilter] = []\n template_variables: List[TemplateVariable] = []\n execution_mappings: List[ExecutionMapping] = []\n if normalized_source_kind == \"superset_link\" and parsed_context is not None:\n recovered_filters, template_variables, execution_mappings, findings = (\n self._build_recovery_bootstrap(\n environment=environment,\n session=persisted_session,\n parsed_context=parsed_context,\n findings=findings,\n )\n )\n\n profile = build_initial_profile(\n session_id=persisted_session.session_id,\n parsed_context=parsed_context,\n dataset_ref=dataset_ref,\n )\n self.repository.event_logger.log_event(\n SessionEventPayload(\n session_id=persisted_session.session_id,\n actor_user_id=command.user.id,\n event_type=\"session_started\",\n event_summary=\"Dataset review session shell created\",\n current_phase=persisted_session.current_phase.value,\n readiness_state=persisted_session.readiness_state.value,\n event_details={\n \"source_kind\": persisted_session.source_kind,\n \"dataset_ref\": persisted_session.dataset_ref,\n \"dataset_id\": persisted_session.dataset_id,\n \"dashboard_id\": persisted_session.dashboard_id,\n \"partial_recovery\": bool(parsed_context and parsed_context.partial_recovery),\n },\n )\n )\n persisted_session = self.repository.save_profile_and_findings(\n persisted_session.session_id,\n command.user.id,\n profile,\n findings,\n )\n if recovered_filters or template_variables or execution_mappings:\n persisted_session = self.repository.save_recovery_state(\n persisted_session.session_id,\n command.user.id,\n recovered_filters,\n template_variables,\n execution_mappings,\n )\n\n active_task_id = self._enqueue_recovery_task(\n command=command,\n session=persisted_session,\n parsed_context=parsed_context,\n )\n if active_task_id:\n persisted_session.active_task_id = active_task_id\n self.repository.bump_session_version(persisted_session)\n self.repository.db.commit()\n self.repository.db.refresh(persisted_session)\n self.repository.event_logger.log_event(\n SessionEventPayload(\n session_id=persisted_session.session_id,\n actor_user_id=command.user.id,\n event_type=\"recovery_task_linked\",\n event_summary=\"Recovery task linked to dataset review session\",\n current_phase=persisted_session.current_phase.value,\n readiness_state=persisted_session.readiness_state.value,\n event_details={\"task_id\": active_task_id},\n )\n )\n logger.reason(\"Linked recovery task to started dataset review session\", extra={\"session_id\": persisted_session.session_id, \"task_id\": active_task_id})\n\n logger.reflect(\"Dataset review session start completed\", extra={\"session_id\": persisted_session.session_id, \"dataset_ref\": persisted_session.dataset_ref, \"readiness_state\": persisted_session.readiness_state.value, \"active_task_id\": persisted_session.active_task_id, \"finding_count\": len(findings)})\n return StartSessionResult(\n session=persisted_session,\n parsed_context=parsed_context,\n findings=findings,\n )\n\n # [/DEF:start_session:Function]\n\n # [DEF:prepare_launch_preview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation.\n # @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview]\n # @PRE: all required variables have candidate values or explicitly accepted defaults.\n # @POST: returns preview artifact in pending, ready, failed, or stale state.\n # @SIDE_EFFECT: persists preview attempt and upstream compilation diagnostics.\n # @DATA_CONTRACT: Input[PreparePreviewCommand] -> Output[PreparePreviewResult]\n def prepare_launch_preview(self, command: PreparePreviewCommand) -> PreparePreviewResult:\n with belief_scope(\"DatasetReviewOrchestrator.prepare_launch_preview\"):\n session = self.repository.load_session_detail(command.session_id, command.user.id)\n if session is None or session.user_id != command.user.id:\n logger.explore(\"Preview preparation rejected because owned session was not found\", extra={\"session_id\": command.session_id, \"user_id\": command.user.id})\n raise ValueError(\"Session not found\")\n\n if command.expected_version is not None:\n self.repository.require_session_version(session, command.expected_version)\n\n if session.dataset_id is None:\n raise ValueError(\"Preview requires a resolved dataset_id\")\n\n environment = self.config_manager.get_environment(session.environment_id)\n if environment is None:\n raise ValueError(\"Environment not found\")\n\n execution_snapshot = build_execution_snapshot(session)\n preview_blockers = execution_snapshot[\"preview_blockers\"]\n if preview_blockers:\n logger.explore(\"Preview preparation blocked by incomplete execution context\", extra={\"session_id\": session.session_id, \"blocked_reasons\": preview_blockers})\n raise ValueError(\"Preview blocked: \" + \"; \".join(preview_blockers))\n\n adapter = SupersetCompilationAdapter(environment)\n preview = adapter.compile_preview(\n PreviewCompilationPayload(\n session_id=session.session_id,\n dataset_id=session.dataset_id,\n preview_fingerprint=execution_snapshot[\"preview_fingerprint\"],\n template_params=execution_snapshot[\"template_params\"],\n effective_filters=execution_snapshot[\"effective_filters\"],\n )\n )\n persisted_preview = self.repository.save_preview(\n session.session_id,\n command.user.id,\n preview,\n expected_version=command.expected_version,\n )\n\n session.current_phase = SessionPhase.PREVIEW\n session.last_activity_at = datetime.utcnow()\n if persisted_preview.preview_status == PreviewStatus.READY:\n launch_blockers = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=persisted_preview)\n if launch_blockers:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n session.recommended_action = RecommendedAction.APPROVE_MAPPING\n else:\n session.readiness_state = ReadinessState.RUN_READY\n session.recommended_action = RecommendedAction.LAUNCH_DATASET\n else:\n session.readiness_state = ReadinessState.PARTIALLY_READY\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n self.repository.db.commit()\n self.repository.db.refresh(session)\n self.repository.event_logger.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=command.user.id,\n event_type=\"preview_generated\",\n event_summary=\"Superset preview generation persisted\",\n current_phase=session.current_phase.value,\n readiness_state=session.readiness_state.value,\n event_details={\"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value, \"preview_fingerprint\": persisted_preview.preview_fingerprint},\n )\n )\n\n logger.reflect(\"Superset preview preparation completed\", extra={\"session_id\": session.session_id, \"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value})\n return PreparePreviewResult(session=session, preview=persisted_preview, blocked_reasons=[])\n\n # [/DEF:prepare_launch_preview:Function]\n\n # [DEF:launch_dataset:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Start the approved dataset execution through SQL Lab and persist run context for audit/replay.\n # @RELATION: CALLS -> [SupersetCompilationAdapter.create_sql_lab_session]\n # @PRE: session is run-ready and compiled preview is current.\n # @POST: returns persisted run context with SQL Lab session reference and launch outcome.\n # @SIDE_EFFECT: creates SQL Lab execution session and audit snapshot.\n # @DATA_CONTRACT: Input[LaunchDatasetCommand] -> Output[LaunchDatasetResult]\n # @INVARIANT: launch remains blocked unless blocking findings are closed, approvals are satisfied, and the latest preview fingerprint matches current execution inputs.\n def launch_dataset(self, command: LaunchDatasetCommand) -> LaunchDatasetResult:\n with belief_scope(\"DatasetReviewOrchestrator.launch_dataset\"):\n session = self.repository.load_session_detail(command.session_id, command.user.id)\n if session is None or session.user_id != command.user.id:\n logger.explore(\"Launch rejected because owned session was not found\", extra={\"session_id\": command.session_id, \"user_id\": command.user.id})\n raise ValueError(\"Session not found\")\n\n if command.expected_version is not None:\n self.repository.require_session_version(session, command.expected_version)\n\n if session.dataset_id is None:\n raise ValueError(\"Launch requires a resolved dataset_id\")\n\n environment = self.config_manager.get_environment(session.environment_id)\n if environment is None:\n raise ValueError(\"Environment not found\")\n\n execution_snapshot = build_execution_snapshot(session)\n current_preview = get_latest_preview(session)\n launch_blockers_list = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=current_preview)\n if launch_blockers_list:\n logger.explore(\"Launch gate blocked dataset execution\", extra={\"session_id\": session.session_id, \"blocked_reasons\": launch_blockers_list})\n raise ValueError(\"Launch blocked: \" + \"; \".join(launch_blockers_list))\n\n adapter = SupersetCompilationAdapter(environment)\n try:\n sql_lab_session_ref = adapter.create_sql_lab_session(\n SqlLabLaunchPayload(\n session_id=session.session_id,\n dataset_id=session.dataset_id,\n preview_id=current_preview.preview_id,\n compiled_sql=str(current_preview.compiled_sql or \"\"),\n template_params=execution_snapshot[\"template_params\"],\n )\n )\n launch_status = LaunchStatus.STARTED\n launch_error = None\n except Exception as exc:\n logger.explore(\"SQL Lab launch failed after passing gates\", extra={\"session_id\": session.session_id, \"error\": str(exc)})\n sql_lab_session_ref = \"unavailable\"\n launch_status = LaunchStatus.FAILED\n launch_error = str(exc)\n\n run_context = DatasetRunContext(\n session_id=session.session_id,\n dataset_ref=session.dataset_ref,\n environment_id=session.environment_id,\n preview_id=current_preview.preview_id,\n sql_lab_session_ref=sql_lab_session_ref,\n effective_filters=execution_snapshot[\"effective_filters\"],\n template_params=execution_snapshot[\"template_params\"],\n approved_mapping_ids=execution_snapshot[\"approved_mapping_ids\"],\n semantic_decision_refs=execution_snapshot[\"semantic_decision_refs\"],\n open_warning_refs=execution_snapshot[\"open_warning_refs\"],\n launch_status=launch_status,\n launch_error=launch_error,\n )\n persisted_run_context = self.repository.save_run_context(\n session.session_id,\n command.user.id,\n run_context,\n expected_version=command.expected_version,\n )\n\n session.current_phase = SessionPhase.LAUNCH\n session.last_activity_at = datetime.utcnow()\n if launch_status == LaunchStatus.FAILED:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n session.recommended_action = RecommendedAction.LAUNCH_DATASET\n else:\n session.readiness_state = ReadinessState.RUN_IN_PROGRESS\n session.recommended_action = RecommendedAction.EXPORT_OUTPUTS\n self.repository.db.commit()\n self.repository.db.refresh(session)\n self.repository.event_logger.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=command.user.id,\n event_type=\"dataset_launch_requested\",\n event_summary=\"Dataset launch handoff persisted\",\n current_phase=session.current_phase.value,\n readiness_state=session.readiness_state.value,\n event_details={\"run_context_id\": persisted_run_context.run_context_id, \"launch_status\": persisted_run_context.launch_status.value, \"preview_id\": persisted_run_context.preview_id, \"sql_lab_session_ref\": persisted_run_context.sql_lab_session_ref},\n )\n )\n\n logger.reflect(\"Dataset launch orchestration completed with audited run context\", extra={\"session_id\": session.session_id, \"run_context_id\": persisted_run_context.run_context_id, \"launch_status\": persisted_run_context.launch_status.value})\n return LaunchDatasetResult(session=session, run_context=persisted_run_context, blocked_reasons=[])\n\n # [/DEF:launch_dataset:Function]\n\n # [DEF:_build_recovery_bootstrap:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation.\n # @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope.\n # @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly.\n # @SIDE_EFFECT: Performs Superset reads through the extractor and may append warning findings for incomplete recovery.\n def _build_recovery_bootstrap(\n self,\n environment,\n session: DatasetReviewSession,\n parsed_context: SupersetParsedContext,\n findings: List[ValidationFinding],\n ) -> tuple[List[ImportedFilter], List[TemplateVariable], List[ExecutionMapping], List[ValidationFinding]]:\n session_record = cast(Any, session)\n extractor = SupersetContextExtractor(environment)\n imported_filters_payload = extractor.recover_imported_filters(parsed_context)\n if imported_filters_payload is None:\n imported_filters_payload = []\n imported_filters = [\n ImportedFilter(\n session_id=session_record.session_id,\n filter_name=str(item.get(\"filter_name\") or f\"imported_filter_{index}\"),\n display_name=item.get(\"display_name\"),\n raw_value=item.get(\"raw_value\"),\n raw_value_masked=bool(item.get(\"raw_value_masked\", False)),\n normalized_value=item.get(\"normalized_value\"),\n source=FilterSource(str(item.get(\"source\") or FilterSource.SUPERSET_URL.value)),\n confidence_state=FilterConfidenceState(str(item.get(\"confidence_state\") or FilterConfidenceState.UNRESOLVED.value)),\n requires_confirmation=bool(item.get(\"requires_confirmation\", False)),\n recovery_status=FilterRecoveryStatus(str(item.get(\"recovery_status\") or FilterRecoveryStatus.PARTIAL.value)),\n notes=item.get(\"notes\"),\n )\n for index, item in enumerate(imported_filters_payload)\n ]\n\n template_variables: List[TemplateVariable] = []\n execution_mappings: List[ExecutionMapping] = []\n\n if session.dataset_id is not None:\n try:\n dataset_payload = parsed_context.dataset_payload\n if not isinstance(dataset_payload, dict):\n dataset_payload = extractor.client.get_dataset_detail(session_record.dataset_id)\n discovered_variables = extractor.discover_template_variables(dataset_payload)\n template_variables = [\n TemplateVariable(\n session_id=session_record.session_id,\n variable_name=str(item.get(\"variable_name\") or f\"variable_{index}\"),\n expression_source=str(item.get(\"expression_source\") or \"\"),\n variable_kind=VariableKind(str(item.get(\"variable_kind\") or VariableKind.UNKNOWN.value)),\n is_required=bool(item.get(\"is_required\", True)),\n default_value=item.get(\"default_value\"),\n mapping_status=MappingStatus(str(item.get(\"mapping_status\") or MappingStatus.UNMAPPED.value)),\n )\n for index, item in enumerate(discovered_variables)\n ]\n except Exception as exc:\n if \"dataset_template_variable_discovery_failed\" not in parsed_context.unresolved_references:\n parsed_context.unresolved_references.append(\"dataset_template_variable_discovery_failed\")\n if not any(f.caused_by_ref == \"dataset_template_variable_discovery_failed\" for f in findings):\n findings.append(\n ValidationFinding(\n area=FindingArea.TEMPLATE_MAPPING,\n severity=FindingSeverity.WARNING,\n code=\"TEMPLATE_VARIABLE_DISCOVERY_FAILED\",\n title=\"Template variables could not be discovered\",\n message=\"Session remains usable, but dataset template variables still need review.\",\n resolution_state=ResolutionState.OPEN,\n caused_by_ref=\"dataset_template_variable_discovery_failed\",\n )\n )\n logger.explore(\"Template variable discovery failed during session bootstrap\", extra={\"session_id\": session_record.session_id, \"dataset_id\": session_record.dataset_id, \"error\": str(exc)})\n\n filter_lookup = {str(f.filter_name or \"\").strip().lower(): f for f in imported_filters if str(f.filter_name or \"\").strip()}\n for tv in template_variables:\n matched_filter = filter_lookup.get(str(tv.variable_name or \"\").strip().lower())\n if matched_filter is None:\n continue\n requires_explicit_approval = bool(matched_filter.requires_confirmation or matched_filter.recovery_status != FilterRecoveryStatus.RECOVERED)\n execution_mappings.append(\n ExecutionMapping(\n session_id=session_record.session_id,\n filter_id=matched_filter.filter_id,\n variable_id=tv.variable_id,\n mapping_method=MappingMethod.DIRECT_MATCH,\n raw_input_value=matched_filter.raw_value,\n effective_value=matched_filter.normalized_value if matched_filter.normalized_value is not None else matched_filter.raw_value,\n transformation_note=\"Bootstrapped from Superset recovery context\",\n warning_level=None,\n requires_explicit_approval=requires_explicit_approval,\n approval_state=ApprovalState.PENDING if requires_explicit_approval else ApprovalState.NOT_REQUIRED,\n approved_by_user_id=None,\n approved_at=None,\n )\n )\n\n return imported_filters, template_variables, execution_mappings, findings\n\n # [/DEF:_build_recovery_bootstrap:Function]\n\n # [DEF:_enqueue_recovery_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Link session start to observable async recovery when task infrastructure is available.\n # @PRE: session is already persisted.\n # @POST: returns task identifier when a task could be enqueued, otherwise None.\n # @SIDE_EFFECT: may create one background task for progressive recovery.\n def _enqueue_recovery_task(\n self,\n command: StartSessionCommand,\n session: DatasetReviewSession,\n parsed_context: Optional[SupersetParsedContext],\n ) -> Optional[str]:\n session_record = cast(Any, session)\n if self.task_manager is None:\n logger.reason(\"Dataset review session started without task manager; continuing synchronously\", extra={\"session_id\": session_record.session_id})\n return None\n\n task_params: Dict[str, Any] = {\n \"session_id\": session_record.session_id,\n \"user_id\": command.user.id,\n \"environment_id\": session_record.environment_id,\n \"source_kind\": session_record.source_kind,\n \"source_input\": session_record.source_input,\n \"dataset_ref\": session_record.dataset_ref,\n \"dataset_id\": session_record.dataset_id,\n \"dashboard_id\": session_record.dashboard_id,\n \"partial_recovery\": bool(parsed_context and parsed_context.partial_recovery),\n }\n\n create_task = getattr(self.task_manager, \"create_task\", None)\n if create_task is None:\n logger.explore(\"Task manager has no create_task method; skipping recovery enqueue\")\n return None\n\n try:\n task_object = create_task(plugin_id=\"dataset-review-recovery\", params=task_params)\n except TypeError:\n logger.explore(\"Recovery task enqueue skipped because task manager create_task contract is incompatible\", extra={\"session_id\": session_record.session_id})\n return None\n\n task_id = getattr(task_object, \"id\", None)\n return str(task_id) if task_id else None\n\n # [/DEF:_enqueue_recovery_task:Function]\n\n\n# [/DEF:DatasetReviewOrchestrator:Class]\n\n# [/DEF:DatasetReviewOrchestrator:Module]\n" + "body": "# [DEF:DatasetReviewOrchestrator:Module]\n# @COMPLEXITY: 5\n# @SEMANTICS: dataset_review, orchestration, session_lifecycle, intake, recovery\n# @PURPOSE: Coordinate dataset review session startup and lifecycle-safe intake recovery for one authenticated user.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: DEPENDS_ON -> [SemanticSourceResolver]\n# @RELATION: DEPENDS_ON -> [SupersetContextExtractor]\n# @RELATION: DEPENDS_ON -> [SupersetCompilationAdapter]\n# @RELATION: DEPENDS_ON -> [TaskManager]\n# @RELATION: DISPATCHES -> [OrchestratorHelpers:Module]\n# @RELATION: DISPATCHES -> [OrchestratorCommands:Module]\n# @PRE: session mutations must execute inside a persisted session boundary scoped to one authenticated user.\n# @POST: state transitions are persisted atomically and emit observable progress for long-running steps.\n# @SIDE_EFFECT: creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts.\n# @DATA_CONTRACT: Input[SessionCommand] -> Output[DatasetReviewSession | CompiledPreview | DatasetRunContext]\n# @INVARIANT: Launch is blocked unless a current session has no open blocking findings, all launch-sensitive mappings are approved, and a non-stale Superset-generated compiled preview matches the current input fingerprint.\n# @RATIONALE: Original 1198-line monolith violated INV_7 (400-line module limit). Decomposed into commands and helpers sub-modules while preserving the orchestrator class as the single entry point.\n# @REJECTED: Keeping all orchestration logic in one file because it exceeded the fractal limit by 3x.\n\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass, field\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional, cast\n\nfrom src.core.config_manager import ConfigManager\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.core.task_manager import TaskManager\nfrom src.core.utils.superset_compilation_adapter import (\n PreviewCompilationPayload,\n SqlLabLaunchPayload,\n SupersetCompilationAdapter,\n)\nfrom src.core.utils.superset_context_extractor import (\n SupersetContextExtractor,\n SupersetParsedContext,\n)\nfrom src.models.auth import User\nfrom src.models.dataset_review import (\n ApprovalState,\n BusinessSummarySource,\n CompiledPreview,\n ConfidenceState,\n DatasetProfile,\n DatasetReviewSession,\n DatasetRunContext,\n ExecutionMapping,\n FilterConfidenceState,\n FilterRecoveryStatus,\n FilterSource,\n FindingArea,\n FindingSeverity,\n ImportedFilter,\n LaunchStatus,\n MappingMethod,\n MappingStatus,\n PreviewStatus,\n RecommendedAction,\n ReadinessState,\n ResolutionState,\n SessionPhase,\n SessionStatus,\n TemplateVariable,\n ValidationFinding,\n VariableKind,\n)\nfrom src.services.dataset_review.repositories.session_repository import (\n DatasetReviewSessionRepository,\n)\nfrom src.services.dataset_review.semantic_resolver import SemanticSourceResolver\nfrom src.services.dataset_review.event_logger import SessionEventPayload\nfrom src.services.dataset_review.orchestrator_pkg._commands import (\n StartSessionCommand,\n StartSessionResult,\n PreparePreviewCommand,\n PreparePreviewResult,\n LaunchDatasetCommand,\n LaunchDatasetResult,\n)\nfrom src.services.dataset_review.orchestrator_pkg._helpers import (\n parse_dataset_selection,\n build_initial_profile,\n build_partial_recovery_findings,\n build_execution_snapshot,\n build_launch_blockers,\n get_latest_preview,\n compute_preview_fingerprint,\n extract_effective_filter_value,\n)\n\nlog = MarkerLogger(\"Orchestrator\")\n\n\n# [DEF:DatasetReviewOrchestrator:Class]\n# @COMPLEXITY: 5\n# @PURPOSE: Coordinate safe session startup while preserving cross-user isolation and explicit partial recovery.\n# @RELATION: DEPENDS_ON -> [DatasetReviewSessionRepository]\n# @RELATION: DEPENDS_ON -> [SupersetContextExtractor]\n# @RELATION: DEPENDS_ON -> [TaskManager]\n# @RELATION: DEPENDS_ON -> [ConfigManager]\n# @RELATION: DEPENDS_ON -> [SemanticSourceResolver]\n# @RELATION: CALLS -> [OrchestratorHelpers:Module]\n# @PRE: constructor dependencies are valid and tied to the current request/task scope.\n# @POST: orchestrator instance can execute session-scoped mutations for one authenticated user.\n# @SIDE_EFFECT: downstream operations may persist session/profile/finding state and enqueue background tasks.\n# @DATA_CONTRACT: Input[StartSessionCommand] -> Output[StartSessionResult]\n# @INVARIANT: session ownership is preserved on every mutation and recovery remains explicit when partial.\nclass DatasetReviewOrchestrator:\n # [DEF:DatasetReviewOrchestrator_init:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary.\n # @PRE: repository/config_manager are valid collaborators for the current request scope.\n # @POST: Instance holds collaborator references used by start/preview/launch orchestration methods.\n def __init__(\n self,\n repository: DatasetReviewSessionRepository,\n config_manager: ConfigManager,\n task_manager: Optional[TaskManager] = None,\n semantic_resolver: Optional[SemanticSourceResolver] = None,\n ) -> None:\n self.repository = repository\n self.config_manager = config_manager\n self.task_manager = task_manager\n self.semantic_resolver = semantic_resolver or SemanticSourceResolver()\n\n # [/DEF:DatasetReviewOrchestrator_init:Function]\n\n # [DEF:start_session:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery.\n # @RELATION: CALLS -> [SupersetContextExtractor.parse_superset_link]\n # @RELATION: CALLS -> [TaskManager.create_task]\n # @PRE: source input is non-empty and environment is accessible.\n # @POST: session exists in persisted storage with intake/recovery state and task linkage when async work is required.\n # @SIDE_EFFECT: persists session and may enqueue recovery task.\n # @DATA_CONTRACT: Input[StartSessionCommand] -> Output[StartSessionResult]\n # @INVARIANT: no cross-user session leakage occurs; session and follow-up task remain owned by the authenticated user.\n def start_session(self, command: StartSessionCommand) -> StartSessionResult:\n with belief_scope(\"DatasetReviewOrchestrator.start_session\"):\n normalized_source_kind = str(command.source_kind or \"\").strip()\n normalized_source_input = str(command.source_input or \"\").strip()\n normalized_environment_id = str(command.environment_id or \"\").strip()\n\n if not normalized_source_input:\n log.explore(\"Blocked dataset review session start due to empty source input\", error=\"source_input was empty\")\n raise ValueError(\"source_input must be non-empty\")\n\n if normalized_source_kind not in {\"superset_link\", \"dataset_selection\"}:\n log.explore(\"Blocked dataset review session start due to unsupported source kind\", payload={\"source_kind\": normalized_source_kind}, error=\"Unsupported source_kind\")\n raise ValueError(\"source_kind must be 'superset_link' or 'dataset_selection'\")\n\n environment = self.config_manager.get_environment(normalized_environment_id)\n if environment is None:\n log.explore(\"Blocked dataset review session start because environment was not found\", payload={\"environment_id\": normalized_environment_id}, error=\"Environment not found in config\")\n raise ValueError(\"Environment not found\")\n\n log.reason(\"Starting dataset review session\", payload={\"user_id\": command.user.id, \"environment_id\": normalized_environment_id, \"source_kind\": normalized_source_kind})\n\n parsed_context: Optional[SupersetParsedContext] = None\n findings: List[ValidationFinding] = []\n dataset_ref = normalized_source_input\n dataset_id: Optional[int] = None\n dashboard_id: Optional[int] = None\n readiness_state = ReadinessState.IMPORTING\n recommended_action = RecommendedAction.REVIEW_DOCUMENTATION\n current_phase = SessionPhase.RECOVERY\n\n if normalized_source_kind == \"superset_link\":\n extractor = SupersetContextExtractor(environment)\n parsed_context = extractor.parse_superset_link(normalized_source_input)\n dataset_ref = parsed_context.dataset_ref\n dataset_id = parsed_context.dataset_id\n dashboard_id = parsed_context.dashboard_id\n\n if parsed_context.partial_recovery:\n readiness_state = ReadinessState.RECOVERY_REQUIRED\n recommended_action = RecommendedAction.REVIEW_DOCUMENTATION\n findings.extend(build_partial_recovery_findings(parsed_context))\n else:\n readiness_state = ReadinessState.REVIEW_READY\n else:\n dataset_ref, dataset_id = parse_dataset_selection(normalized_source_input)\n readiness_state = ReadinessState.REVIEW_READY\n current_phase = SessionPhase.REVIEW\n\n session = DatasetReviewSession(\n user_id=command.user.id,\n environment_id=normalized_environment_id,\n source_kind=normalized_source_kind,\n source_input=normalized_source_input,\n dataset_ref=dataset_ref,\n dataset_id=dataset_id,\n dashboard_id=dashboard_id,\n readiness_state=readiness_state,\n recommended_action=recommended_action,\n status=SessionStatus.ACTIVE,\n current_phase=current_phase,\n )\n persisted_session = cast(Any, self.repository.create_session(session))\n\n recovered_filters: List[ImportedFilter] = []\n template_variables: List[TemplateVariable] = []\n execution_mappings: List[ExecutionMapping] = []\n if normalized_source_kind == \"superset_link\" and parsed_context is not None:\n recovered_filters, template_variables, execution_mappings, findings = (\n self._build_recovery_bootstrap(\n environment=environment,\n session=persisted_session,\n parsed_context=parsed_context,\n findings=findings,\n )\n )\n\n profile = build_initial_profile(\n session_id=persisted_session.session_id,\n parsed_context=parsed_context,\n dataset_ref=dataset_ref,\n )\n self.repository.event_log.log_event(\n SessionEventPayload(\n session_id=persisted_session.session_id,\n actor_user_id=command.user.id,\n event_type=\"session_started\",\n event_summary=\"Dataset review session shell created\",\n current_phase=persisted_session.current_phase.value,\n readiness_state=persisted_session.readiness_state.value,\n event_details={\n \"source_kind\": persisted_session.source_kind,\n \"dataset_ref\": persisted_session.dataset_ref,\n \"dataset_id\": persisted_session.dataset_id,\n \"dashboard_id\": persisted_session.dashboard_id,\n \"partial_recovery\": bool(parsed_context and parsed_context.partial_recovery),\n },\n )\n )\n persisted_session = self.repository.save_profile_and_findings(\n persisted_session.session_id,\n command.user.id,\n profile,\n findings,\n )\n if recovered_filters or template_variables or execution_mappings:\n persisted_session = self.repository.save_recovery_state(\n persisted_session.session_id,\n command.user.id,\n recovered_filters,\n template_variables,\n execution_mappings,\n )\n\n active_task_id = self._enqueue_recovery_task(\n command=command,\n session=persisted_session,\n parsed_context=parsed_context,\n )\n if active_task_id:\n persisted_session.active_task_id = active_task_id\n self.repository.bump_session_version(persisted_session)\n self.repository.db.commit()\n self.repository.db.refresh(persisted_session)\n self.repository.event_log.log_event(\n SessionEventPayload(\n session_id=persisted_session.session_id,\n actor_user_id=command.user.id,\n event_type=\"recovery_task_linked\",\n event_summary=\"Recovery task linked to dataset review session\",\n current_phase=persisted_session.current_phase.value,\n readiness_state=persisted_session.readiness_state.value,\n event_details={\"task_id\": active_task_id},\n )\n )\n log.reason(\"Linked recovery task to started dataset review session\", payload={\"session_id\": persisted_session.session_id, \"task_id\": active_task_id})\n\n log.reflect(\"Dataset review session start completed\", payload={\"session_id\": persisted_session.session_id, \"dataset_ref\": persisted_session.dataset_ref, \"readiness_state\": persisted_session.readiness_state.value, \"active_task_id\": persisted_session.active_task_id, \"finding_count\": len(findings)})\n return StartSessionResult(\n session=persisted_session,\n parsed_context=parsed_context,\n findings=findings,\n )\n\n # [/DEF:start_session:Function]\n\n # [DEF:prepare_launch_preview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation.\n # @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview]\n # @PRE: all required variables have candidate values or explicitly accepted defaults.\n # @POST: returns preview artifact in pending, ready, failed, or stale state.\n # @SIDE_EFFECT: persists preview attempt and upstream compilation diagnostics.\n # @DATA_CONTRACT: Input[PreparePreviewCommand] -> Output[PreparePreviewResult]\n def prepare_launch_preview(self, command: PreparePreviewCommand) -> PreparePreviewResult:\n with belief_scope(\"DatasetReviewOrchestrator.prepare_launch_preview\"):\n session = self.repository.load_session_detail(command.session_id, command.user.id)\n if session is None or session.user_id != command.user.id:\n log.explore(\"Preview preparation rejected because owned session was not found\", payload={\"session_id\": command.session_id, \"user_id\": command.user.id}, error=\"Session not found or access denied\")\n raise ValueError(\"Session not found\")\n\n if command.expected_version is not None:\n self.repository.require_session_version(session, command.expected_version)\n\n if session.dataset_id is None:\n raise ValueError(\"Preview requires a resolved dataset_id\")\n\n environment = self.config_manager.get_environment(session.environment_id)\n if environment is None:\n raise ValueError(\"Environment not found\")\n\n execution_snapshot = build_execution_snapshot(session)\n preview_blockers = execution_snapshot[\"preview_blockers\"]\n if preview_blockers:\n log.explore(\"Preview preparation blocked by incomplete execution context\", payload={\"session_id\": session.session_id, \"blocked_reasons\": preview_blockers}, error=\"Preview blockers present\")\n raise ValueError(\"Preview blocked: \" + \"; \".join(preview_blockers))\n\n adapter = SupersetCompilationAdapter(environment)\n preview = adapter.compile_preview(\n PreviewCompilationPayload(\n session_id=session.session_id,\n dataset_id=session.dataset_id,\n preview_fingerprint=execution_snapshot[\"preview_fingerprint\"],\n template_params=execution_snapshot[\"template_params\"],\n effective_filters=execution_snapshot[\"effective_filters\"],\n )\n )\n persisted_preview = self.repository.save_preview(\n session.session_id,\n command.user.id,\n preview,\n expected_version=command.expected_version,\n )\n\n session.current_phase = SessionPhase.PREVIEW\n session.last_activity_at = datetime.utcnow()\n if persisted_preview.preview_status == PreviewStatus.READY:\n launch_blockers = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=persisted_preview)\n if launch_blockers:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n session.recommended_action = RecommendedAction.APPROVE_MAPPING\n else:\n session.readiness_state = ReadinessState.RUN_READY\n session.recommended_action = RecommendedAction.LAUNCH_DATASET\n else:\n session.readiness_state = ReadinessState.PARTIALLY_READY\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n self.repository.db.commit()\n self.repository.db.refresh(session)\n self.repository.event_log.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=command.user.id,\n event_type=\"preview_generated\",\n event_summary=\"Superset preview generation persisted\",\n current_phase=session.current_phase.value,\n readiness_state=session.readiness_state.value,\n event_details={\"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value, \"preview_fingerprint\": persisted_preview.preview_fingerprint},\n )\n )\n\n log.reflect(\"Superset preview preparation completed\", payload={\"session_id\": session.session_id, \"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value})\n return PreparePreviewResult(session=session, preview=persisted_preview, blocked_reasons=[])\n\n # [/DEF:prepare_launch_preview:Function]\n\n # [DEF:launch_dataset:Function]\n # @COMPLEXITY: 5\n # @PURPOSE: Start the approved dataset execution through SQL Lab and persist run context for audit/replay.\n # @RELATION: CALLS -> [SupersetCompilationAdapter.create_sql_lab_session]\n # @PRE: session is run-ready and compiled preview is current.\n # @POST: returns persisted run context with SQL Lab session reference and launch outcome.\n # @SIDE_EFFECT: creates SQL Lab execution session and audit snapshot.\n # @DATA_CONTRACT: Input[LaunchDatasetCommand] -> Output[LaunchDatasetResult]\n # @INVARIANT: launch remains blocked unless blocking findings are closed, approvals are satisfied, and the latest preview fingerprint matches current execution inputs.\n def launch_dataset(self, command: LaunchDatasetCommand) -> LaunchDatasetResult:\n with belief_scope(\"DatasetReviewOrchestrator.launch_dataset\"):\n session = self.repository.load_session_detail(command.session_id, command.user.id)\n if session is None or session.user_id != command.user.id:\n log.explore(\"Launch rejected because owned session was not found\", payload={\"session_id\": command.session_id, \"user_id\": command.user.id}, error=\"Session not found or access denied\")\n raise ValueError(\"Session not found\")\n\n if command.expected_version is not None:\n self.repository.require_session_version(session, command.expected_version)\n\n if session.dataset_id is None:\n raise ValueError(\"Launch requires a resolved dataset_id\")\n\n environment = self.config_manager.get_environment(session.environment_id)\n if environment is None:\n raise ValueError(\"Environment not found\")\n\n execution_snapshot = build_execution_snapshot(session)\n current_preview = get_latest_preview(session)\n launch_blockers_list = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=current_preview)\n if launch_blockers_list:\n log.explore(\"Launch gate blocked dataset execution\", payload={\"session_id\": session.session_id, \"blocked_reasons\": launch_blockers_list}, error=\"Launch blockers present\")\n raise ValueError(\"Launch blocked: \" + \"; \".join(launch_blockers_list))\n\n adapter = SupersetCompilationAdapter(environment)\n try:\n sql_lab_session_ref = adapter.create_sql_lab_session(\n SqlLabLaunchPayload(\n session_id=session.session_id,\n dataset_id=session.dataset_id,\n preview_id=current_preview.preview_id,\n compiled_sql=str(current_preview.compiled_sql or \"\"),\n template_params=execution_snapshot[\"template_params\"],\n )\n )\n launch_status = LaunchStatus.STARTED\n launch_error = None\n except Exception as exc:\n log.explore(\"SQL Lab launch failed after passing gates\", payload={\"session_id\": session.session_id}, error=str(exc))\n sql_lab_session_ref = \"unavailable\"\n launch_status = LaunchStatus.FAILED\n launch_error = str(exc)\n\n run_context = DatasetRunContext(\n session_id=session.session_id,\n dataset_ref=session.dataset_ref,\n environment_id=session.environment_id,\n preview_id=current_preview.preview_id,\n sql_lab_session_ref=sql_lab_session_ref,\n effective_filters=execution_snapshot[\"effective_filters\"],\n template_params=execution_snapshot[\"template_params\"],\n approved_mapping_ids=execution_snapshot[\"approved_mapping_ids\"],\n semantic_decision_refs=execution_snapshot[\"semantic_decision_refs\"],\n open_warning_refs=execution_snapshot[\"open_warning_refs\"],\n launch_status=launch_status,\n launch_error=launch_error,\n )\n persisted_run_context = self.repository.save_run_context(\n session.session_id,\n command.user.id,\n run_context,\n expected_version=command.expected_version,\n )\n\n session.current_phase = SessionPhase.LAUNCH\n session.last_activity_at = datetime.utcnow()\n if launch_status == LaunchStatus.FAILED:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n session.recommended_action = RecommendedAction.LAUNCH_DATASET\n else:\n session.readiness_state = ReadinessState.RUN_IN_PROGRESS\n session.recommended_action = RecommendedAction.EXPORT_OUTPUTS\n self.repository.db.commit()\n self.repository.db.refresh(session)\n self.repository.event_log.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=command.user.id,\n event_type=\"dataset_launch_requested\",\n event_summary=\"Dataset launch handoff persisted\",\n current_phase=session.current_phase.value,\n readiness_state=session.readiness_state.value,\n event_details={\"run_context_id\": persisted_run_context.run_context_id, \"launch_status\": persisted_run_context.launch_status.value, \"preview_id\": persisted_run_context.preview_id, \"sql_lab_session_ref\": persisted_run_context.sql_lab_session_ref},\n )\n )\n\n log.reflect(\"Dataset launch orchestration completed with audited run context\", payload={\"session_id\": session.session_id, \"run_context_id\": persisted_run_context.run_context_id, \"launch_status\": persisted_run_context.launch_status.value})\n return LaunchDatasetResult(session=session, run_context=persisted_run_context, blocked_reasons=[])\n\n # [/DEF:launch_dataset:Function]\n\n # [DEF:_build_recovery_bootstrap:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation.\n # @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope.\n # @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly.\n # @SIDE_EFFECT: Performs Superset reads through the extractor and may append warning findings for incomplete recovery.\n def _build_recovery_bootstrap(\n self,\n environment,\n session: DatasetReviewSession,\n parsed_context: SupersetParsedContext,\n findings: List[ValidationFinding],\n ) -> tuple[List[ImportedFilter], List[TemplateVariable], List[ExecutionMapping], List[ValidationFinding]]:\n session_record = cast(Any, session)\n extractor = SupersetContextExtractor(environment)\n imported_filters_payload = extractor.recover_imported_filters(parsed_context)\n if imported_filters_payload is None:\n imported_filters_payload = []\n imported_filters = [\n ImportedFilter(\n session_id=session_record.session_id,\n filter_name=str(item.get(\"filter_name\") or f\"imported_filter_{index}\"),\n display_name=item.get(\"display_name\"),\n raw_value=item.get(\"raw_value\"),\n raw_value_masked=bool(item.get(\"raw_value_masked\", False)),\n normalized_value=item.get(\"normalized_value\"),\n source=FilterSource(str(item.get(\"source\") or FilterSource.SUPERSET_URL.value)),\n confidence_state=FilterConfidenceState(str(item.get(\"confidence_state\") or FilterConfidenceState.UNRESOLVED.value)),\n requires_confirmation=bool(item.get(\"requires_confirmation\", False)),\n recovery_status=FilterRecoveryStatus(str(item.get(\"recovery_status\") or FilterRecoveryStatus.PARTIAL.value)),\n notes=item.get(\"notes\"),\n )\n for index, item in enumerate(imported_filters_payload)\n ]\n\n template_variables: List[TemplateVariable] = []\n execution_mappings: List[ExecutionMapping] = []\n\n if session.dataset_id is not None:\n try:\n dataset_payload = parsed_context.dataset_payload\n if not isinstance(dataset_payload, dict):\n dataset_payload = extractor.client.get_dataset_detail(session_record.dataset_id)\n discovered_variables = extractor.discover_template_variables(dataset_payload)\n template_variables = [\n TemplateVariable(\n session_id=session_record.session_id,\n variable_name=str(item.get(\"variable_name\") or f\"variable_{index}\"),\n expression_source=str(item.get(\"expression_source\") or \"\"),\n variable_kind=VariableKind(str(item.get(\"variable_kind\") or VariableKind.UNKNOWN.value)),\n is_required=bool(item.get(\"is_required\", True)),\n default_value=item.get(\"default_value\"),\n mapping_status=MappingStatus(str(item.get(\"mapping_status\") or MappingStatus.UNMAPPED.value)),\n )\n for index, item in enumerate(discovered_variables)\n ]\n except Exception as exc:\n if \"dataset_template_variable_discovery_failed\" not in parsed_context.unresolved_references:\n parsed_context.unresolved_references.append(\"dataset_template_variable_discovery_failed\")\n if not any(f.caused_by_ref == \"dataset_template_variable_discovery_failed\" for f in findings):\n findings.append(\n ValidationFinding(\n area=FindingArea.TEMPLATE_MAPPING,\n severity=FindingSeverity.WARNING,\n code=\"TEMPLATE_VARIABLE_DISCOVERY_FAILED\",\n title=\"Template variables could not be discovered\",\n message=\"Session remains usable, but dataset template variables still need review.\",\n resolution_state=ResolutionState.OPEN,\n caused_by_ref=\"dataset_template_variable_discovery_failed\",\n )\n )\n log.explore(\"Template variable discovery failed during session bootstrap\", payload={\"session_id\": session_record.session_id, \"dataset_id\": session_record.dataset_id}, error=str(exc))\n\n filter_lookup = {str(f.filter_name or \"\").strip().lower(): f for f in imported_filters if str(f.filter_name or \"\").strip()}\n for tv in template_variables:\n matched_filter = filter_lookup.get(str(tv.variable_name or \"\").strip().lower())\n if matched_filter is None:\n continue\n requires_explicit_approval = bool(matched_filter.requires_confirmation or matched_filter.recovery_status != FilterRecoveryStatus.RECOVERED)\n execution_mappings.append(\n ExecutionMapping(\n session_id=session_record.session_id,\n filter_id=matched_filter.filter_id,\n variable_id=tv.variable_id,\n mapping_method=MappingMethod.DIRECT_MATCH,\n raw_input_value=matched_filter.raw_value,\n effective_value=matched_filter.normalized_value if matched_filter.normalized_value is not None else matched_filter.raw_value,\n transformation_note=\"Bootstrapped from Superset recovery context\",\n warning_level=None,\n requires_explicit_approval=requires_explicit_approval,\n approval_state=ApprovalState.PENDING if requires_explicit_approval else ApprovalState.NOT_REQUIRED,\n approved_by_user_id=None,\n approved_at=None,\n )\n )\n\n return imported_filters, template_variables, execution_mappings, findings\n\n # [/DEF:_build_recovery_bootstrap:Function]\n\n # [DEF:_enqueue_recovery_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Link session start to observable async recovery when task infrastructure is available.\n # @PRE: session is already persisted.\n # @POST: returns task identifier when a task could be enqueued, otherwise None.\n # @SIDE_EFFECT: may create one background task for progressive recovery.\n def _enqueue_recovery_task(\n self,\n command: StartSessionCommand,\n session: DatasetReviewSession,\n parsed_context: Optional[SupersetParsedContext],\n ) -> Optional[str]:\n session_record = cast(Any, session)\n if self.task_manager is None:\n log.reason(\"Dataset review session started without task manager; continuing synchronously\", payload={\"session_id\": session_record.session_id})\n return None\n\n task_params: Dict[str, Any] = {\n \"session_id\": session_record.session_id,\n \"user_id\": command.user.id,\n \"environment_id\": session_record.environment_id,\n \"source_kind\": session_record.source_kind,\n \"source_input\": session_record.source_input,\n \"dataset_ref\": session_record.dataset_ref,\n \"dataset_id\": session_record.dataset_id,\n \"dashboard_id\": session_record.dashboard_id,\n \"partial_recovery\": bool(parsed_context and parsed_context.partial_recovery),\n }\n\n create_task = getattr(self.task_manager, \"create_task\", None)\n if create_task is None:\n log.explore(\"Task manager has no create_task method; skipping recovery enqueue\", error=\"create_task method not found\")\n return None\n\n try:\n task_object = create_task(plugin_id=\"dataset-review-recovery\", params=task_params)\n except TypeError:\n log.explore(\"Recovery task enqueue skipped because task manager create_task contract is incompatible\", payload={\"session_id\": session_record.session_id}, error=\"TypeError from create_task\")\n return None\n\n task_id = getattr(task_object, \"id\", None)\n return str(task_id) if task_id else None\n\n # [/DEF:_enqueue_recovery_task:Function]\n\n\n# [/DEF:DatasetReviewOrchestrator:Class]\n\n# [/DEF:DatasetReviewOrchestrator:Module]\n" }, { "contract_id": "DatasetReviewOrchestrator_init", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator.py", - "start_line": 110, - "end_line": 127, + "start_line": 111, + "end_line": 128, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -81531,8 +82540,8 @@ "contract_id": "prepare_launch_preview", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator.py", - "start_line": 284, - "end_line": 362, + "start_line": 285, + "end_line": 363, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81563,14 +82572,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:prepare_launch_preview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation.\n # @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview]\n # @PRE: all required variables have candidate values or explicitly accepted defaults.\n # @POST: returns preview artifact in pending, ready, failed, or stale state.\n # @SIDE_EFFECT: persists preview attempt and upstream compilation diagnostics.\n # @DATA_CONTRACT: Input[PreparePreviewCommand] -> Output[PreparePreviewResult]\n def prepare_launch_preview(self, command: PreparePreviewCommand) -> PreparePreviewResult:\n with belief_scope(\"DatasetReviewOrchestrator.prepare_launch_preview\"):\n session = self.repository.load_session_detail(command.session_id, command.user.id)\n if session is None or session.user_id != command.user.id:\n logger.explore(\"Preview preparation rejected because owned session was not found\", extra={\"session_id\": command.session_id, \"user_id\": command.user.id})\n raise ValueError(\"Session not found\")\n\n if command.expected_version is not None:\n self.repository.require_session_version(session, command.expected_version)\n\n if session.dataset_id is None:\n raise ValueError(\"Preview requires a resolved dataset_id\")\n\n environment = self.config_manager.get_environment(session.environment_id)\n if environment is None:\n raise ValueError(\"Environment not found\")\n\n execution_snapshot = build_execution_snapshot(session)\n preview_blockers = execution_snapshot[\"preview_blockers\"]\n if preview_blockers:\n logger.explore(\"Preview preparation blocked by incomplete execution context\", extra={\"session_id\": session.session_id, \"blocked_reasons\": preview_blockers})\n raise ValueError(\"Preview blocked: \" + \"; \".join(preview_blockers))\n\n adapter = SupersetCompilationAdapter(environment)\n preview = adapter.compile_preview(\n PreviewCompilationPayload(\n session_id=session.session_id,\n dataset_id=session.dataset_id,\n preview_fingerprint=execution_snapshot[\"preview_fingerprint\"],\n template_params=execution_snapshot[\"template_params\"],\n effective_filters=execution_snapshot[\"effective_filters\"],\n )\n )\n persisted_preview = self.repository.save_preview(\n session.session_id,\n command.user.id,\n preview,\n expected_version=command.expected_version,\n )\n\n session.current_phase = SessionPhase.PREVIEW\n session.last_activity_at = datetime.utcnow()\n if persisted_preview.preview_status == PreviewStatus.READY:\n launch_blockers = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=persisted_preview)\n if launch_blockers:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n session.recommended_action = RecommendedAction.APPROVE_MAPPING\n else:\n session.readiness_state = ReadinessState.RUN_READY\n session.recommended_action = RecommendedAction.LAUNCH_DATASET\n else:\n session.readiness_state = ReadinessState.PARTIALLY_READY\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n self.repository.db.commit()\n self.repository.db.refresh(session)\n self.repository.event_logger.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=command.user.id,\n event_type=\"preview_generated\",\n event_summary=\"Superset preview generation persisted\",\n current_phase=session.current_phase.value,\n readiness_state=session.readiness_state.value,\n event_details={\"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value, \"preview_fingerprint\": persisted_preview.preview_fingerprint},\n )\n )\n\n logger.reflect(\"Superset preview preparation completed\", extra={\"session_id\": session.session_id, \"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value})\n return PreparePreviewResult(session=session, preview=persisted_preview, blocked_reasons=[])\n\n # [/DEF:prepare_launch_preview:Function]\n" + "body": " # [DEF:prepare_launch_preview:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation.\n # @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview]\n # @PRE: all required variables have candidate values or explicitly accepted defaults.\n # @POST: returns preview artifact in pending, ready, failed, or stale state.\n # @SIDE_EFFECT: persists preview attempt and upstream compilation diagnostics.\n # @DATA_CONTRACT: Input[PreparePreviewCommand] -> Output[PreparePreviewResult]\n def prepare_launch_preview(self, command: PreparePreviewCommand) -> PreparePreviewResult:\n with belief_scope(\"DatasetReviewOrchestrator.prepare_launch_preview\"):\n session = self.repository.load_session_detail(command.session_id, command.user.id)\n if session is None or session.user_id != command.user.id:\n log.explore(\"Preview preparation rejected because owned session was not found\", payload={\"session_id\": command.session_id, \"user_id\": command.user.id}, error=\"Session not found or access denied\")\n raise ValueError(\"Session not found\")\n\n if command.expected_version is not None:\n self.repository.require_session_version(session, command.expected_version)\n\n if session.dataset_id is None:\n raise ValueError(\"Preview requires a resolved dataset_id\")\n\n environment = self.config_manager.get_environment(session.environment_id)\n if environment is None:\n raise ValueError(\"Environment not found\")\n\n execution_snapshot = build_execution_snapshot(session)\n preview_blockers = execution_snapshot[\"preview_blockers\"]\n if preview_blockers:\n log.explore(\"Preview preparation blocked by incomplete execution context\", payload={\"session_id\": session.session_id, \"blocked_reasons\": preview_blockers}, error=\"Preview blockers present\")\n raise ValueError(\"Preview blocked: \" + \"; \".join(preview_blockers))\n\n adapter = SupersetCompilationAdapter(environment)\n preview = adapter.compile_preview(\n PreviewCompilationPayload(\n session_id=session.session_id,\n dataset_id=session.dataset_id,\n preview_fingerprint=execution_snapshot[\"preview_fingerprint\"],\n template_params=execution_snapshot[\"template_params\"],\n effective_filters=execution_snapshot[\"effective_filters\"],\n )\n )\n persisted_preview = self.repository.save_preview(\n session.session_id,\n command.user.id,\n preview,\n expected_version=command.expected_version,\n )\n\n session.current_phase = SessionPhase.PREVIEW\n session.last_activity_at = datetime.utcnow()\n if persisted_preview.preview_status == PreviewStatus.READY:\n launch_blockers = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=persisted_preview)\n if launch_blockers:\n session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY\n session.recommended_action = RecommendedAction.APPROVE_MAPPING\n else:\n session.readiness_state = ReadinessState.RUN_READY\n session.recommended_action = RecommendedAction.LAUNCH_DATASET\n else:\n session.readiness_state = ReadinessState.PARTIALLY_READY\n session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW\n self.repository.db.commit()\n self.repository.db.refresh(session)\n self.repository.event_log.log_event(\n SessionEventPayload(\n session_id=session.session_id,\n actor_user_id=command.user.id,\n event_type=\"preview_generated\",\n event_summary=\"Superset preview generation persisted\",\n current_phase=session.current_phase.value,\n readiness_state=session.readiness_state.value,\n event_details={\"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value, \"preview_fingerprint\": persisted_preview.preview_fingerprint},\n )\n )\n\n log.reflect(\"Superset preview preparation completed\", payload={\"session_id\": session.session_id, \"preview_id\": persisted_preview.preview_id, \"preview_status\": persisted_preview.preview_status.value})\n return PreparePreviewResult(session=session, preview=persisted_preview, blocked_reasons=[])\n\n # [/DEF:prepare_launch_preview:Function]\n" }, { "contract_id": "_build_recovery_bootstrap", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator.py", - "start_line": 464, - "end_line": 562, + "start_line": 465, + "end_line": 563, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81593,14 +82602,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_build_recovery_bootstrap:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation.\n # @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope.\n # @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly.\n # @SIDE_EFFECT: Performs Superset reads through the extractor and may append warning findings for incomplete recovery.\n def _build_recovery_bootstrap(\n self,\n environment,\n session: DatasetReviewSession,\n parsed_context: SupersetParsedContext,\n findings: List[ValidationFinding],\n ) -> tuple[List[ImportedFilter], List[TemplateVariable], List[ExecutionMapping], List[ValidationFinding]]:\n session_record = cast(Any, session)\n extractor = SupersetContextExtractor(environment)\n imported_filters_payload = extractor.recover_imported_filters(parsed_context)\n if imported_filters_payload is None:\n imported_filters_payload = []\n imported_filters = [\n ImportedFilter(\n session_id=session_record.session_id,\n filter_name=str(item.get(\"filter_name\") or f\"imported_filter_{index}\"),\n display_name=item.get(\"display_name\"),\n raw_value=item.get(\"raw_value\"),\n raw_value_masked=bool(item.get(\"raw_value_masked\", False)),\n normalized_value=item.get(\"normalized_value\"),\n source=FilterSource(str(item.get(\"source\") or FilterSource.SUPERSET_URL.value)),\n confidence_state=FilterConfidenceState(str(item.get(\"confidence_state\") or FilterConfidenceState.UNRESOLVED.value)),\n requires_confirmation=bool(item.get(\"requires_confirmation\", False)),\n recovery_status=FilterRecoveryStatus(str(item.get(\"recovery_status\") or FilterRecoveryStatus.PARTIAL.value)),\n notes=item.get(\"notes\"),\n )\n for index, item in enumerate(imported_filters_payload)\n ]\n\n template_variables: List[TemplateVariable] = []\n execution_mappings: List[ExecutionMapping] = []\n\n if session.dataset_id is not None:\n try:\n dataset_payload = parsed_context.dataset_payload\n if not isinstance(dataset_payload, dict):\n dataset_payload = extractor.client.get_dataset_detail(session_record.dataset_id)\n discovered_variables = extractor.discover_template_variables(dataset_payload)\n template_variables = [\n TemplateVariable(\n session_id=session_record.session_id,\n variable_name=str(item.get(\"variable_name\") or f\"variable_{index}\"),\n expression_source=str(item.get(\"expression_source\") or \"\"),\n variable_kind=VariableKind(str(item.get(\"variable_kind\") or VariableKind.UNKNOWN.value)),\n is_required=bool(item.get(\"is_required\", True)),\n default_value=item.get(\"default_value\"),\n mapping_status=MappingStatus(str(item.get(\"mapping_status\") or MappingStatus.UNMAPPED.value)),\n )\n for index, item in enumerate(discovered_variables)\n ]\n except Exception as exc:\n if \"dataset_template_variable_discovery_failed\" not in parsed_context.unresolved_references:\n parsed_context.unresolved_references.append(\"dataset_template_variable_discovery_failed\")\n if not any(f.caused_by_ref == \"dataset_template_variable_discovery_failed\" for f in findings):\n findings.append(\n ValidationFinding(\n area=FindingArea.TEMPLATE_MAPPING,\n severity=FindingSeverity.WARNING,\n code=\"TEMPLATE_VARIABLE_DISCOVERY_FAILED\",\n title=\"Template variables could not be discovered\",\n message=\"Session remains usable, but dataset template variables still need review.\",\n resolution_state=ResolutionState.OPEN,\n caused_by_ref=\"dataset_template_variable_discovery_failed\",\n )\n )\n logger.explore(\"Template variable discovery failed during session bootstrap\", extra={\"session_id\": session_record.session_id, \"dataset_id\": session_record.dataset_id, \"error\": str(exc)})\n\n filter_lookup = {str(f.filter_name or \"\").strip().lower(): f for f in imported_filters if str(f.filter_name or \"\").strip()}\n for tv in template_variables:\n matched_filter = filter_lookup.get(str(tv.variable_name or \"\").strip().lower())\n if matched_filter is None:\n continue\n requires_explicit_approval = bool(matched_filter.requires_confirmation or matched_filter.recovery_status != FilterRecoveryStatus.RECOVERED)\n execution_mappings.append(\n ExecutionMapping(\n session_id=session_record.session_id,\n filter_id=matched_filter.filter_id,\n variable_id=tv.variable_id,\n mapping_method=MappingMethod.DIRECT_MATCH,\n raw_input_value=matched_filter.raw_value,\n effective_value=matched_filter.normalized_value if matched_filter.normalized_value is not None else matched_filter.raw_value,\n transformation_note=\"Bootstrapped from Superset recovery context\",\n warning_level=None,\n requires_explicit_approval=requires_explicit_approval,\n approval_state=ApprovalState.PENDING if requires_explicit_approval else ApprovalState.NOT_REQUIRED,\n approved_by_user_id=None,\n approved_at=None,\n )\n )\n\n return imported_filters, template_variables, execution_mappings, findings\n\n # [/DEF:_build_recovery_bootstrap:Function]\n" + "body": " # [DEF:_build_recovery_bootstrap:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation.\n # @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope.\n # @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly.\n # @SIDE_EFFECT: Performs Superset reads through the extractor and may append warning findings for incomplete recovery.\n def _build_recovery_bootstrap(\n self,\n environment,\n session: DatasetReviewSession,\n parsed_context: SupersetParsedContext,\n findings: List[ValidationFinding],\n ) -> tuple[List[ImportedFilter], List[TemplateVariable], List[ExecutionMapping], List[ValidationFinding]]:\n session_record = cast(Any, session)\n extractor = SupersetContextExtractor(environment)\n imported_filters_payload = extractor.recover_imported_filters(parsed_context)\n if imported_filters_payload is None:\n imported_filters_payload = []\n imported_filters = [\n ImportedFilter(\n session_id=session_record.session_id,\n filter_name=str(item.get(\"filter_name\") or f\"imported_filter_{index}\"),\n display_name=item.get(\"display_name\"),\n raw_value=item.get(\"raw_value\"),\n raw_value_masked=bool(item.get(\"raw_value_masked\", False)),\n normalized_value=item.get(\"normalized_value\"),\n source=FilterSource(str(item.get(\"source\") or FilterSource.SUPERSET_URL.value)),\n confidence_state=FilterConfidenceState(str(item.get(\"confidence_state\") or FilterConfidenceState.UNRESOLVED.value)),\n requires_confirmation=bool(item.get(\"requires_confirmation\", False)),\n recovery_status=FilterRecoveryStatus(str(item.get(\"recovery_status\") or FilterRecoveryStatus.PARTIAL.value)),\n notes=item.get(\"notes\"),\n )\n for index, item in enumerate(imported_filters_payload)\n ]\n\n template_variables: List[TemplateVariable] = []\n execution_mappings: List[ExecutionMapping] = []\n\n if session.dataset_id is not None:\n try:\n dataset_payload = parsed_context.dataset_payload\n if not isinstance(dataset_payload, dict):\n dataset_payload = extractor.client.get_dataset_detail(session_record.dataset_id)\n discovered_variables = extractor.discover_template_variables(dataset_payload)\n template_variables = [\n TemplateVariable(\n session_id=session_record.session_id,\n variable_name=str(item.get(\"variable_name\") or f\"variable_{index}\"),\n expression_source=str(item.get(\"expression_source\") or \"\"),\n variable_kind=VariableKind(str(item.get(\"variable_kind\") or VariableKind.UNKNOWN.value)),\n is_required=bool(item.get(\"is_required\", True)),\n default_value=item.get(\"default_value\"),\n mapping_status=MappingStatus(str(item.get(\"mapping_status\") or MappingStatus.UNMAPPED.value)),\n )\n for index, item in enumerate(discovered_variables)\n ]\n except Exception as exc:\n if \"dataset_template_variable_discovery_failed\" not in parsed_context.unresolved_references:\n parsed_context.unresolved_references.append(\"dataset_template_variable_discovery_failed\")\n if not any(f.caused_by_ref == \"dataset_template_variable_discovery_failed\" for f in findings):\n findings.append(\n ValidationFinding(\n area=FindingArea.TEMPLATE_MAPPING,\n severity=FindingSeverity.WARNING,\n code=\"TEMPLATE_VARIABLE_DISCOVERY_FAILED\",\n title=\"Template variables could not be discovered\",\n message=\"Session remains usable, but dataset template variables still need review.\",\n resolution_state=ResolutionState.OPEN,\n caused_by_ref=\"dataset_template_variable_discovery_failed\",\n )\n )\n log.explore(\"Template variable discovery failed during session bootstrap\", payload={\"session_id\": session_record.session_id, \"dataset_id\": session_record.dataset_id}, error=str(exc))\n\n filter_lookup = {str(f.filter_name or \"\").strip().lower(): f for f in imported_filters if str(f.filter_name or \"\").strip()}\n for tv in template_variables:\n matched_filter = filter_lookup.get(str(tv.variable_name or \"\").strip().lower())\n if matched_filter is None:\n continue\n requires_explicit_approval = bool(matched_filter.requires_confirmation or matched_filter.recovery_status != FilterRecoveryStatus.RECOVERED)\n execution_mappings.append(\n ExecutionMapping(\n session_id=session_record.session_id,\n filter_id=matched_filter.filter_id,\n variable_id=tv.variable_id,\n mapping_method=MappingMethod.DIRECT_MATCH,\n raw_input_value=matched_filter.raw_value,\n effective_value=matched_filter.normalized_value if matched_filter.normalized_value is not None else matched_filter.raw_value,\n transformation_note=\"Bootstrapped from Superset recovery context\",\n warning_level=None,\n requires_explicit_approval=requires_explicit_approval,\n approval_state=ApprovalState.PENDING if requires_explicit_approval else ApprovalState.NOT_REQUIRED,\n approved_by_user_id=None,\n approved_at=None,\n )\n )\n\n return imported_filters, template_variables, execution_mappings, findings\n\n # [/DEF:_build_recovery_bootstrap:Function]\n" }, { "contract_id": "_enqueue_recovery_task", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator.py", - "start_line": 564, - "end_line": 607, + "start_line": 565, + "end_line": 608, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -81650,7 +82659,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_enqueue_recovery_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Link session start to observable async recovery when task infrastructure is available.\n # @PRE: session is already persisted.\n # @POST: returns task identifier when a task could be enqueued, otherwise None.\n # @SIDE_EFFECT: may create one background task for progressive recovery.\n def _enqueue_recovery_task(\n self,\n command: StartSessionCommand,\n session: DatasetReviewSession,\n parsed_context: Optional[SupersetParsedContext],\n ) -> Optional[str]:\n session_record = cast(Any, session)\n if self.task_manager is None:\n logger.reason(\"Dataset review session started without task manager; continuing synchronously\", extra={\"session_id\": session_record.session_id})\n return None\n\n task_params: Dict[str, Any] = {\n \"session_id\": session_record.session_id,\n \"user_id\": command.user.id,\n \"environment_id\": session_record.environment_id,\n \"source_kind\": session_record.source_kind,\n \"source_input\": session_record.source_input,\n \"dataset_ref\": session_record.dataset_ref,\n \"dataset_id\": session_record.dataset_id,\n \"dashboard_id\": session_record.dashboard_id,\n \"partial_recovery\": bool(parsed_context and parsed_context.partial_recovery),\n }\n\n create_task = getattr(self.task_manager, \"create_task\", None)\n if create_task is None:\n logger.explore(\"Task manager has no create_task method; skipping recovery enqueue\")\n return None\n\n try:\n task_object = create_task(plugin_id=\"dataset-review-recovery\", params=task_params)\n except TypeError:\n logger.explore(\"Recovery task enqueue skipped because task manager create_task contract is incompatible\", extra={\"session_id\": session_record.session_id})\n return None\n\n task_id = getattr(task_object, \"id\", None)\n return str(task_id) if task_id else None\n\n # [/DEF:_enqueue_recovery_task:Function]\n" + "body": " # [DEF:_enqueue_recovery_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Link session start to observable async recovery when task infrastructure is available.\n # @PRE: session is already persisted.\n # @POST: returns task identifier when a task could be enqueued, otherwise None.\n # @SIDE_EFFECT: may create one background task for progressive recovery.\n def _enqueue_recovery_task(\n self,\n command: StartSessionCommand,\n session: DatasetReviewSession,\n parsed_context: Optional[SupersetParsedContext],\n ) -> Optional[str]:\n session_record = cast(Any, session)\n if self.task_manager is None:\n log.reason(\"Dataset review session started without task manager; continuing synchronously\", payload={\"session_id\": session_record.session_id})\n return None\n\n task_params: Dict[str, Any] = {\n \"session_id\": session_record.session_id,\n \"user_id\": command.user.id,\n \"environment_id\": session_record.environment_id,\n \"source_kind\": session_record.source_kind,\n \"source_input\": session_record.source_input,\n \"dataset_ref\": session_record.dataset_ref,\n \"dataset_id\": session_record.dataset_id,\n \"dashboard_id\": session_record.dashboard_id,\n \"partial_recovery\": bool(parsed_context and parsed_context.partial_recovery),\n }\n\n create_task = getattr(self.task_manager, \"create_task\", None)\n if create_task is None:\n log.explore(\"Task manager has no create_task method; skipping recovery enqueue\", error=\"create_task method not found\")\n return None\n\n try:\n task_object = create_task(plugin_id=\"dataset-review-recovery\", params=task_params)\n except TypeError:\n log.explore(\"Recovery task enqueue skipped because task manager create_task contract is incompatible\", payload={\"session_id\": session_record.session_id}, error=\"TypeError from create_task\")\n return None\n\n task_id = getattr(task_object, \"id\", None)\n return str(task_id) if task_id else None\n\n # [/DEF:_enqueue_recovery_task:Function]\n" }, { "contract_id": "OrchestratorCommands", @@ -81800,7 +82809,7 @@ "contract_type": "Module", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", "start_line": 1, - "end_line": 356, + "end_line": 355, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81836,14 +82845,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:OrchestratorHelpers:Module]\n# @COMPLEXITY: 4\n# @PURPOSE: Pure helper methods extracted from DatasetReviewOrchestrator for INV_7 compliance — snapshot, blockers, fingerprint, recovery bootstrap.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewModels]\n# @RELATION: DEPENDS_ON -> [SupersetContextExtractor]\n# @PRE: Caller provides a loaded session aggregate with hydrated child collections.\n# @POST: Helper results are deterministic and do not mutate persistence directly.\n\nfrom __future__ import annotations\n\nimport hashlib\nimport json\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional, cast\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.dataset_review import (\n ApprovalState,\n CompiledPreview,\n ConfidenceState,\n DatasetProfile,\n DatasetReviewSession,\n ExecutionMapping,\n FilterConfidenceState,\n FilterRecoveryStatus,\n FilterSource,\n FindingArea,\n FindingSeverity,\n ImportedFilter,\n MappingMethod,\n MappingStatus,\n PreviewStatus,\n ResolutionState,\n TemplateVariable,\n ValidationFinding,\n VariableKind,\n BusinessSummarySource,\n)\n\nlogger = cast(Any, logger)\n\n\n# [DEF:parse_dataset_selection:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Normalize dataset-selection payload into canonical session references.\ndef parse_dataset_selection(source_input: str) -> tuple[str, Optional[int]]:\n normalized = str(source_input or \"\").strip()\n if not normalized:\n raise ValueError(\"dataset selection input must be non-empty\")\n if normalized.isdigit():\n dataset_id = int(normalized)\n return f\"dataset:{dataset_id}\", dataset_id\n if normalized.startswith(\"dataset:\"):\n suffix = normalized.split(\":\", 1)[1].strip()\n if suffix.isdigit():\n return normalized, int(suffix)\n return normalized, None\n return normalized, None\n\n\n# [/DEF:parse_dataset_selection:Function]\n\n\n# [DEF:build_initial_profile:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Create the first profile snapshot so exports and detail views remain usable immediately after intake.\ndef build_initial_profile(\n session_id: str,\n parsed_context: Optional[Any],\n dataset_ref: str,\n) -> DatasetProfile:\n dataset_name = (\n dataset_ref.split(\".\")[-1] if dataset_ref else \"Unresolved dataset\"\n )\n business_summary = (\n f\"Review session initialized for {dataset_ref}.\"\n if dataset_ref\n else \"Review session initialized with unresolved dataset context.\"\n )\n confidence_state = (\n ConfidenceState.MIXED\n if parsed_context and getattr(parsed_context, \"partial_recovery\", False)\n else ConfidenceState.MOSTLY_CONFIRMED\n )\n return DatasetProfile(\n session_id=session_id,\n dataset_name=dataset_name or \"Unresolved dataset\",\n schema_name=dataset_ref.split(\".\")[0] if \".\" in dataset_ref else None,\n business_summary=business_summary,\n business_summary_source=BusinessSummarySource.IMPORTED,\n description=\"Initial review profile created from source intake.\",\n dataset_type=\"unknown\",\n is_sqllab_view=False,\n completeness_score=0.25,\n confidence_state=confidence_state,\n has_blocking_findings=False,\n has_warning_findings=bool(\n parsed_context and getattr(parsed_context, \"partial_recovery\", False)\n ),\n manual_summary_locked=False,\n )\n\n\n# [/DEF:build_initial_profile:Function]\n\n\n# [DEF:build_partial_recovery_findings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Project partial Superset intake recovery into explicit findings without blocking session usability.\n# @PRE: parsed_context.partial_recovery is true.\n# @POST: Returns warning-level findings that preserve usable but incomplete state.\ndef build_partial_recovery_findings(parsed_context: Any) -> List[ValidationFinding]:\n findings: List[ValidationFinding] = []\n for unresolved_ref in getattr(parsed_context, \"unresolved_references\", []):\n findings.append(\n ValidationFinding(\n area=FindingArea.SOURCE_INTAKE,\n severity=FindingSeverity.WARNING,\n code=\"PARTIAL_SUPERSET_RECOVERY\",\n title=\"Superset context recovered partially\",\n message=(\n \"Session remains usable, but some Superset context requires review: \"\n f\"{unresolved_ref.replace('_', ' ')}.\"\n ),\n resolution_state=ResolutionState.OPEN,\n caused_by_ref=unresolved_ref,\n )\n )\n return findings\n\n\n# [/DEF:build_partial_recovery_findings:Function]\n\n\n# [DEF:extract_effective_filter_value:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Separate normalized filter payload metadata from the user-facing effective filter value.\ndef extract_effective_filter_value(\n normalized_value: Any, raw_value: Any\n) -> Any:\n if isinstance(normalized_value, dict) and (\n \"filter_clauses\" in normalized_value\n or \"extra_form_data\" in normalized_value\n ):\n return raw_value\n return normalized_value if normalized_value is not None else raw_value\n\n\n# [/DEF:extract_effective_filter_value:Function]\n\n\n# [DEF:build_execution_snapshot:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Build effective filters, template params, approvals, and fingerprint for preview and launch gating.\n# @PRE: Session aggregate includes imported filters, template variables, and current execution mappings.\n# @POST: Returns deterministic execution snapshot for current session state without mutating persistence.\ndef build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]:\n session_record = cast(Any, session)\n filter_lookup = {\n item.filter_id: item for item in session_record.imported_filters\n }\n variable_lookup = {\n item.variable_id: item for item in session_record.template_variables\n }\n\n effective_filters: List[Dict[str, Any]] = []\n template_params: Dict[str, Any] = {}\n approved_mapping_ids: List[str] = []\n open_warning_refs: List[str] = []\n preview_blockers: List[str] = []\n mapped_filter_ids: set[str] = set()\n\n for mapping in session_record.execution_mappings:\n imported_filter = filter_lookup.get(mapping.filter_id)\n template_variable = variable_lookup.get(mapping.variable_id)\n if imported_filter is None:\n preview_blockers.append(f\"mapping:{mapping.mapping_id}:missing_filter\")\n continue\n if template_variable is None:\n preview_blockers.append(f\"mapping:{mapping.mapping_id}:missing_variable\")\n continue\n\n effective_value = mapping.effective_value\n if effective_value is None:\n effective_value = extract_effective_filter_value(\n imported_filter.normalized_value, imported_filter.raw_value,\n )\n if effective_value is None:\n effective_value = template_variable.default_value\n\n if effective_value is None and template_variable.is_required:\n preview_blockers.append(\n f\"variable:{template_variable.variable_name}:missing_required_value\"\n )\n continue\n\n mapped_filter_ids.add(imported_filter.filter_id)\n if effective_value is not None:\n mapped_filter_payload = {\n \"mapping_id\": mapping.mapping_id,\n \"filter_id\": imported_filter.filter_id,\n \"filter_name\": imported_filter.filter_name,\n \"variable_id\": template_variable.variable_id,\n \"variable_name\": template_variable.variable_name,\n \"effective_value\": effective_value,\n \"raw_input_value\": mapping.raw_input_value,\n }\n if isinstance(imported_filter.normalized_value, dict):\n mapped_filter_payload[\"display_name\"] = imported_filter.display_name\n mapped_filter_payload[\"normalized_filter_payload\"] = (\n imported_filter.normalized_value\n )\n effective_filters.append(mapped_filter_payload)\n template_params[template_variable.variable_name] = effective_value\n if mapping.approval_state == ApprovalState.APPROVED:\n approved_mapping_ids.append(mapping.mapping_id)\n if (\n mapping.requires_explicit_approval\n and mapping.approval_state != ApprovalState.APPROVED\n ):\n open_warning_refs.append(mapping.mapping_id)\n\n for imported_filter in session_record.imported_filters:\n if imported_filter.filter_id in mapped_filter_ids:\n continue\n effective_value = extract_effective_filter_value(\n imported_filter.normalized_value, imported_filter.raw_value,\n )\n if effective_value is None:\n continue\n effective_filters.append(\n {\n \"filter_id\": imported_filter.filter_id,\n \"filter_name\": imported_filter.filter_name,\n \"display_name\": imported_filter.display_name,\n \"effective_value\": effective_value,\n \"raw_input_value\": imported_filter.raw_value,\n \"normalized_filter_payload\": imported_filter.normalized_value,\n }\n )\n\n mapped_variable_ids = {\n mapping.variable_id for mapping in session_record.execution_mappings\n }\n for variable in session_record.template_variables:\n if variable.variable_id in mapped_variable_ids:\n continue\n if variable.default_value is not None:\n template_params[variable.variable_name] = variable.default_value\n continue\n if variable.is_required:\n preview_blockers.append(f\"variable:{variable.variable_name}:unmapped\")\n\n semantic_decision_refs = [\n field.field_id\n for field in session.semantic_fields\n if field.is_locked\n or not field.needs_review\n or field.provenance.value != \"unresolved\"\n ]\n preview_fingerprint = compute_preview_fingerprint(\n {\n \"dataset_id\": session_record.dataset_id,\n \"template_params\": template_params,\n \"effective_filters\": effective_filters,\n }\n )\n return {\n \"effective_filters\": effective_filters,\n \"template_params\": template_params,\n \"approved_mapping_ids\": sorted(approved_mapping_ids),\n \"semantic_decision_refs\": sorted(semantic_decision_refs),\n \"open_warning_refs\": sorted(open_warning_refs),\n \"preview_blockers\": sorted(set(preview_blockers)),\n \"preview_fingerprint\": preview_fingerprint,\n }\n\n\n# [/DEF:build_execution_snapshot:Function]\n\n\n# [DEF:build_launch_blockers:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Enforce launch gates from findings, approvals, and current preview truth.\n# @PRE: execution_snapshot was computed from current session state.\n# @POST: Returns explicit blocker codes for every unmet launch invariant.\ndef build_launch_blockers(\n session: DatasetReviewSession,\n execution_snapshot: Dict[str, Any],\n preview: Optional[CompiledPreview],\n) -> List[str]:\n session_record = cast(Any, session)\n blockers = list(execution_snapshot[\"preview_blockers\"])\n\n for finding in session_record.findings:\n if (\n finding.severity == FindingSeverity.BLOCKING\n and finding.resolution_state\n not in {ResolutionState.RESOLVED, ResolutionState.APPROVED}\n ):\n blockers.append(f\"finding:{finding.code}:blocking\")\n for mapping in session_record.execution_mappings:\n if (\n mapping.requires_explicit_approval\n and mapping.approval_state != ApprovalState.APPROVED\n ):\n blockers.append(f\"mapping:{mapping.mapping_id}:approval_required\")\n\n if preview is None:\n blockers.append(\"preview:missing\")\n else:\n if preview.preview_status != PreviewStatus.READY:\n blockers.append(f\"preview:{preview.preview_status.value}\")\n if preview.preview_fingerprint != execution_snapshot[\"preview_fingerprint\"]:\n blockers.append(\"preview:fingerprint_mismatch\")\n\n return sorted(set(blockers))\n\n\n# [/DEF:build_launch_blockers:Function]\n\n\n# [DEF:get_latest_preview:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve the current latest preview snapshot for one session aggregate.\ndef get_latest_preview(session: DatasetReviewSession) -> Optional[CompiledPreview]:\n session_record = cast(Any, session)\n if not session_record.previews:\n return None\n if session_record.last_preview_id:\n for preview in session_record.previews:\n if preview.preview_id == session_record.last_preview_id:\n return preview\n return sorted(\n session_record.previews,\n key=lambda item: (item.created_at or datetime.min, item.preview_id),\n reverse=True,\n )[0]\n\n\n# [/DEF:get_latest_preview:Function]\n\n\n# [DEF:compute_preview_fingerprint:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Produce deterministic execution fingerprint for preview truth and staleness checks.\ndef compute_preview_fingerprint(payload: Dict[str, Any]) -> str:\n serialized = json.dumps(payload, sort_keys=True, default=str)\n return hashlib.sha256(serialized.encode(\"utf-8\")).hexdigest()\n\n\n# [/DEF:compute_preview_fingerprint:Function]\n\n\n# [/DEF:OrchestratorHelpers:Module]\n" + "body": "# [DEF:OrchestratorHelpers:Module]\n# @COMPLEXITY: 4\n# @PURPOSE: Pure helper methods extracted from DatasetReviewOrchestrator for INV_7 compliance — snapshot, blockers, fingerprint, recovery bootstrap.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewModels]\n# @RELATION: DEPENDS_ON -> [SupersetContextExtractor]\n# @PRE: Caller provides a loaded session aggregate with hydrated child collections.\n# @POST: Helper results are deterministic and do not mutate persistence directly.\n\nfrom __future__ import annotations\n\nimport hashlib\nimport json\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional, cast\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.models.dataset_review import (\n ApprovalState,\n CompiledPreview,\n ConfidenceState,\n DatasetProfile,\n DatasetReviewSession,\n ExecutionMapping,\n FilterConfidenceState,\n FilterRecoveryStatus,\n FilterSource,\n FindingArea,\n FindingSeverity,\n ImportedFilter,\n MappingMethod,\n MappingStatus,\n PreviewStatus,\n ResolutionState,\n TemplateVariable,\n ValidationFinding,\n VariableKind,\n BusinessSummarySource,\n)\n\nlog = MarkerLogger(\"OrchestratorHelpers\")\n\n# [DEF:parse_dataset_selection:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Normalize dataset-selection payload into canonical session references.\ndef parse_dataset_selection(source_input: str) -> tuple[str, Optional[int]]:\n normalized = str(source_input or \"\").strip()\n if not normalized:\n raise ValueError(\"dataset selection input must be non-empty\")\n if normalized.isdigit():\n dataset_id = int(normalized)\n return f\"dataset:{dataset_id}\", dataset_id\n if normalized.startswith(\"dataset:\"):\n suffix = normalized.split(\":\", 1)[1].strip()\n if suffix.isdigit():\n return normalized, int(suffix)\n return normalized, None\n return normalized, None\n\n\n# [/DEF:parse_dataset_selection:Function]\n\n\n# [DEF:build_initial_profile:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Create the first profile snapshot so exports and detail views remain usable immediately after intake.\ndef build_initial_profile(\n session_id: str,\n parsed_context: Optional[Any],\n dataset_ref: str,\n) -> DatasetProfile:\n dataset_name = (\n dataset_ref.split(\".\")[-1] if dataset_ref else \"Unresolved dataset\"\n )\n business_summary = (\n f\"Review session initialized for {dataset_ref}.\"\n if dataset_ref\n else \"Review session initialized with unresolved dataset context.\"\n )\n confidence_state = (\n ConfidenceState.MIXED\n if parsed_context and getattr(parsed_context, \"partial_recovery\", False)\n else ConfidenceState.MOSTLY_CONFIRMED\n )\n return DatasetProfile(\n session_id=session_id,\n dataset_name=dataset_name or \"Unresolved dataset\",\n schema_name=dataset_ref.split(\".\")[0] if \".\" in dataset_ref else None,\n business_summary=business_summary,\n business_summary_source=BusinessSummarySource.IMPORTED,\n description=\"Initial review profile created from source intake.\",\n dataset_type=\"unknown\",\n is_sqllab_view=False,\n completeness_score=0.25,\n confidence_state=confidence_state,\n has_blocking_findings=False,\n has_warning_findings=bool(\n parsed_context and getattr(parsed_context, \"partial_recovery\", False)\n ),\n manual_summary_locked=False,\n )\n\n\n# [/DEF:build_initial_profile:Function]\n\n\n# [DEF:build_partial_recovery_findings:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Project partial Superset intake recovery into explicit findings without blocking session usability.\n# @PRE: parsed_context.partial_recovery is true.\n# @POST: Returns warning-level findings that preserve usable but incomplete state.\ndef build_partial_recovery_findings(parsed_context: Any) -> List[ValidationFinding]:\n findings: List[ValidationFinding] = []\n for unresolved_ref in getattr(parsed_context, \"unresolved_references\", []):\n findings.append(\n ValidationFinding(\n area=FindingArea.SOURCE_INTAKE,\n severity=FindingSeverity.WARNING,\n code=\"PARTIAL_SUPERSET_RECOVERY\",\n title=\"Superset context recovered partially\",\n message=(\n \"Session remains usable, but some Superset context requires review: \"\n f\"{unresolved_ref.replace('_', ' ')}.\"\n ),\n resolution_state=ResolutionState.OPEN,\n caused_by_ref=unresolved_ref,\n )\n )\n return findings\n\n\n# [/DEF:build_partial_recovery_findings:Function]\n\n\n# [DEF:extract_effective_filter_value:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Separate normalized filter payload metadata from the user-facing effective filter value.\ndef extract_effective_filter_value(\n normalized_value: Any, raw_value: Any\n) -> Any:\n if isinstance(normalized_value, dict) and (\n \"filter_clauses\" in normalized_value\n or \"extra_form_data\" in normalized_value\n ):\n return raw_value\n return normalized_value if normalized_value is not None else raw_value\n\n\n# [/DEF:extract_effective_filter_value:Function]\n\n\n# [DEF:build_execution_snapshot:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Build effective filters, template params, approvals, and fingerprint for preview and launch gating.\n# @PRE: Session aggregate includes imported filters, template variables, and current execution mappings.\n# @POST: Returns deterministic execution snapshot for current session state without mutating persistence.\ndef build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]:\n session_record = cast(Any, session)\n filter_lookup = {\n item.filter_id: item for item in session_record.imported_filters\n }\n variable_lookup = {\n item.variable_id: item for item in session_record.template_variables\n }\n\n effective_filters: List[Dict[str, Any]] = []\n template_params: Dict[str, Any] = {}\n approved_mapping_ids: List[str] = []\n open_warning_refs: List[str] = []\n preview_blockers: List[str] = []\n mapped_filter_ids: set[str] = set()\n\n for mapping in session_record.execution_mappings:\n imported_filter = filter_lookup.get(mapping.filter_id)\n template_variable = variable_lookup.get(mapping.variable_id)\n if imported_filter is None:\n preview_blockers.append(f\"mapping:{mapping.mapping_id}:missing_filter\")\n continue\n if template_variable is None:\n preview_blockers.append(f\"mapping:{mapping.mapping_id}:missing_variable\")\n continue\n\n effective_value = mapping.effective_value\n if effective_value is None:\n effective_value = extract_effective_filter_value(\n imported_filter.normalized_value, imported_filter.raw_value,\n )\n if effective_value is None:\n effective_value = template_variable.default_value\n\n if effective_value is None and template_variable.is_required:\n preview_blockers.append(\n f\"variable:{template_variable.variable_name}:missing_required_value\"\n )\n continue\n\n mapped_filter_ids.add(imported_filter.filter_id)\n if effective_value is not None:\n mapped_filter_payload = {\n \"mapping_id\": mapping.mapping_id,\n \"filter_id\": imported_filter.filter_id,\n \"filter_name\": imported_filter.filter_name,\n \"variable_id\": template_variable.variable_id,\n \"variable_name\": template_variable.variable_name,\n \"effective_value\": effective_value,\n \"raw_input_value\": mapping.raw_input_value,\n }\n if isinstance(imported_filter.normalized_value, dict):\n mapped_filter_payload[\"display_name\"] = imported_filter.display_name\n mapped_filter_payload[\"normalized_filter_payload\"] = (\n imported_filter.normalized_value\n )\n effective_filters.append(mapped_filter_payload)\n template_params[template_variable.variable_name] = effective_value\n if mapping.approval_state == ApprovalState.APPROVED:\n approved_mapping_ids.append(mapping.mapping_id)\n if (\n mapping.requires_explicit_approval\n and mapping.approval_state != ApprovalState.APPROVED\n ):\n open_warning_refs.append(mapping.mapping_id)\n\n for imported_filter in session_record.imported_filters:\n if imported_filter.filter_id in mapped_filter_ids:\n continue\n effective_value = extract_effective_filter_value(\n imported_filter.normalized_value, imported_filter.raw_value,\n )\n if effective_value is None:\n continue\n effective_filters.append(\n {\n \"filter_id\": imported_filter.filter_id,\n \"filter_name\": imported_filter.filter_name,\n \"display_name\": imported_filter.display_name,\n \"effective_value\": effective_value,\n \"raw_input_value\": imported_filter.raw_value,\n \"normalized_filter_payload\": imported_filter.normalized_value,\n }\n )\n\n mapped_variable_ids = {\n mapping.variable_id for mapping in session_record.execution_mappings\n }\n for variable in session_record.template_variables:\n if variable.variable_id in mapped_variable_ids:\n continue\n if variable.default_value is not None:\n template_params[variable.variable_name] = variable.default_value\n continue\n if variable.is_required:\n preview_blockers.append(f\"variable:{variable.variable_name}:unmapped\")\n\n semantic_decision_refs = [\n field.field_id\n for field in session.semantic_fields\n if field.is_locked\n or not field.needs_review\n or field.provenance.value != \"unresolved\"\n ]\n preview_fingerprint = compute_preview_fingerprint(\n {\n \"dataset_id\": session_record.dataset_id,\n \"template_params\": template_params,\n \"effective_filters\": effective_filters,\n }\n )\n return {\n \"effective_filters\": effective_filters,\n \"template_params\": template_params,\n \"approved_mapping_ids\": sorted(approved_mapping_ids),\n \"semantic_decision_refs\": sorted(semantic_decision_refs),\n \"open_warning_refs\": sorted(open_warning_refs),\n \"preview_blockers\": sorted(set(preview_blockers)),\n \"preview_fingerprint\": preview_fingerprint,\n }\n\n\n# [/DEF:build_execution_snapshot:Function]\n\n\n# [DEF:build_launch_blockers:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Enforce launch gates from findings, approvals, and current preview truth.\n# @PRE: execution_snapshot was computed from current session state.\n# @POST: Returns explicit blocker codes for every unmet launch invariant.\ndef build_launch_blockers(\n session: DatasetReviewSession,\n execution_snapshot: Dict[str, Any],\n preview: Optional[CompiledPreview],\n) -> List[str]:\n session_record = cast(Any, session)\n blockers = list(execution_snapshot[\"preview_blockers\"])\n\n for finding in session_record.findings:\n if (\n finding.severity == FindingSeverity.BLOCKING\n and finding.resolution_state\n not in {ResolutionState.RESOLVED, ResolutionState.APPROVED}\n ):\n blockers.append(f\"finding:{finding.code}:blocking\")\n for mapping in session_record.execution_mappings:\n if (\n mapping.requires_explicit_approval\n and mapping.approval_state != ApprovalState.APPROVED\n ):\n blockers.append(f\"mapping:{mapping.mapping_id}:approval_required\")\n\n if preview is None:\n blockers.append(\"preview:missing\")\n else:\n if preview.preview_status != PreviewStatus.READY:\n blockers.append(f\"preview:{preview.preview_status.value}\")\n if preview.preview_fingerprint != execution_snapshot[\"preview_fingerprint\"]:\n blockers.append(\"preview:fingerprint_mismatch\")\n\n return sorted(set(blockers))\n\n\n# [/DEF:build_launch_blockers:Function]\n\n\n# [DEF:get_latest_preview:Function]\n# @COMPLEXITY: 2\n# @PURPOSE: Resolve the current latest preview snapshot for one session aggregate.\ndef get_latest_preview(session: DatasetReviewSession) -> Optional[CompiledPreview]:\n session_record = cast(Any, session)\n if not session_record.previews:\n return None\n if session_record.last_preview_id:\n for preview in session_record.previews:\n if preview.preview_id == session_record.last_preview_id:\n return preview\n return sorted(\n session_record.previews,\n key=lambda item: (item.created_at or datetime.min, item.preview_id),\n reverse=True,\n )[0]\n\n\n# [/DEF:get_latest_preview:Function]\n\n\n# [DEF:compute_preview_fingerprint:Function]\n# @COMPLEXITY: 1\n# @PURPOSE: Produce deterministic execution fingerprint for preview truth and staleness checks.\ndef compute_preview_fingerprint(payload: Dict[str, Any]) -> str:\n serialized = json.dumps(payload, sort_keys=True, default=str)\n return hashlib.sha256(serialized.encode(\"utf-8\")).hexdigest()\n\n\n# [/DEF:compute_preview_fingerprint:Function]\n\n\n# [/DEF:OrchestratorHelpers:Module]\n" }, { "contract_id": "parse_dataset_selection", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 44, - "end_line": 62, + "start_line": 43, + "end_line": 61, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -81859,8 +82868,8 @@ "contract_id": "build_initial_profile", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 65, - "end_line": 105, + "start_line": 64, + "end_line": 104, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -81876,8 +82885,8 @@ "contract_id": "build_partial_recovery_findings", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 108, - "end_line": 133, + "start_line": 107, + "end_line": 132, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -81923,8 +82932,8 @@ "contract_id": "extract_effective_filter_value", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 136, - "end_line": 150, + "start_line": 135, + "end_line": 149, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -81940,8 +82949,8 @@ "contract_id": "build_execution_snapshot", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 153, - "end_line": 280, + "start_line": 152, + "end_line": 279, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -81978,8 +82987,8 @@ "contract_id": "build_launch_blockers", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 283, - "end_line": 321, + "start_line": 282, + "end_line": 320, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -82025,8 +83034,8 @@ "contract_id": "get_latest_preview", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 324, - "end_line": 342, + "start_line": 323, + "end_line": 341, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -82042,8 +83051,8 @@ "contract_id": "compute_preview_fingerprint", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/orchestrator_pkg/_helpers.py", - "start_line": 345, - "end_line": 353, + "start_line": 344, + "end_line": 352, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -82550,7 +83559,7 @@ "contract_type": "Module", "file_path": "backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py", "start_line": 1, - "end_line": 202, + "end_line": 203, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -82586,14 +83595,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SessionRepositoryMutations:Module]\n# @COMPLEXITY: 4\n# @PURPOSE: Persistence mutation operations for dataset review session aggregates — profile/findings, recovery state, preview, run context.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewModels]\n# @RELATION: DEPENDS_ON -> [SessionEventLogger]\n# @PRE: All mutations execute within authenticated request or task scope.\n# @POST: Session aggregate writes preserve ownership and version semantics.\n\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import Any, List, Optional, cast\n\nfrom sqlalchemy.orm import Session\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.dataset_review import (\n ClarificationQuestion,\n ClarificationSession,\n CompiledPreview,\n DatasetProfile,\n DatasetReviewSession,\n DatasetRunContext,\n ExecutionMapping,\n ImportedFilter,\n SemanticFieldEntry,\n SessionCollaborator,\n SessionEvent,\n TemplateVariable,\n ValidationFinding,\n)\nfrom src.services.dataset_review.event_logger import SessionEventLogger\n\nlogger = cast(Any, logger)\n\n\n# [DEF:save_profile_and_findings:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist profile state and replace validation findings for an owned session in one transaction.\n# @PRE: session_id belongs to user_id and the supplied profile/findings belong to the same aggregate scope.\n# @POST: stored profile matches the current session and findings are replaced by the supplied collection.\n# @SIDE_EFFECT: updates profile rows, deletes stale findings, inserts current findings, and commits the transaction.\ndef save_profile_and_findings(\n db: Session,\n event_logger: SessionEventLogger,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n session_id: str,\n user_id: str,\n profile: DatasetProfile,\n findings: List[ValidationFinding],\n expected_version: Optional[int] = None,\n) -> DatasetReviewSession:\n with belief_scope(\"save_profile_and_findings\"):\n session = get_owned_session(session_id, user_id)\n if expected_version is not None:\n require_session_version(session, expected_version)\n logger.reason(\"Persisting dataset profile and replacing validation findings\", extra={\"session_id\": session_id, \"user_id\": user_id, \"has_profile\": bool(profile), \"findings_count\": len(findings)})\n\n if profile:\n existing_profile = db.query(DatasetProfile).filter_by(session_id=session_id).first()\n if existing_profile:\n profile.profile_id = existing_profile.profile_id\n db.merge(profile)\n\n db.query(ValidationFinding).filter(ValidationFinding.session_id == session_id).delete()\n for finding in findings:\n cast(Any, finding).session_id = session_id\n db.add(finding)\n\n commit_session_mutation(session, expected_version=expected_version)\n logger.reflect(\"Dataset profile and validation findings committed\", extra={\"session_id\": session.session_id, \"user_id\": user_id, \"findings_count\": len(findings)})\n\n from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository\n return session\n\n\n# [/DEF:save_profile_and_findings:Function]\n\n\n# [DEF:save_recovery_state:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist imported filters, template variables, and initial execution mappings for one owned session.\n# @PRE: session_id belongs to user_id.\n# @POST: Recovery state persisted to database.\n# @SIDE_EFFECT: Writes to database.\ndef save_recovery_state(\n db: Session,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n load_session_detail_fn,\n session_id: str,\n user_id: str,\n imported_filters: List[ImportedFilter],\n template_variables: List[TemplateVariable],\n execution_mappings: List[ExecutionMapping],\n expected_version: Optional[int] = None,\n) -> DatasetReviewSession:\n with belief_scope(\"save_recovery_state\"):\n session = get_owned_session(session_id, user_id)\n if expected_version is not None:\n require_session_version(session, expected_version)\n logger.reason(\"Persisting dataset review recovery bootstrap state\", extra={\"session_id\": session_id, \"user_id\": user_id, \"imported_filters_count\": len(imported_filters), \"template_variables_count\": len(template_variables), \"execution_mappings_count\": len(execution_mappings)})\n\n db.query(ExecutionMapping).filter(ExecutionMapping.session_id == session_id).delete()\n db.query(TemplateVariable).filter(TemplateVariable.session_id == session_id).delete()\n db.query(ImportedFilter).filter(ImportedFilter.session_id == session_id).delete()\n\n for f in imported_filters:\n cast(Any, f).session_id = session_id\n db.add(f)\n for tv in template_variables:\n cast(Any, tv).session_id = session_id\n db.add(tv)\n db.flush()\n for em in execution_mappings:\n cast(Any, em).session_id = session_id\n db.add(em)\n\n commit_session_mutation(session, expected_version=expected_version)\n logger.reflect(\"Dataset review recovery bootstrap state committed\", extra={\"session_id\": session.session_id, \"user_id\": user_id})\n return load_session_detail_fn(session_id, user_id)\n\n\n# [/DEF:save_recovery_state:Function]\n\n\n# [DEF:save_preview:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist a preview snapshot and mark prior session previews stale.\n# @PRE: session_id belongs to user_id and preview is prepared for the same session aggregate.\n# @POST: preview is persisted and the session points to the latest preview identifier.\n# @SIDE_EFFECT: updates prior preview statuses, inserts a preview row, mutates the parent session, and commits.\ndef save_preview(\n db: Session,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n session_id: str,\n user_id: str,\n preview: CompiledPreview,\n expected_version: Optional[int] = None,\n) -> CompiledPreview:\n with belief_scope(\"save_preview\"):\n session = get_owned_session(session_id, user_id)\n session_record = cast(Any, session)\n if expected_version is not None:\n require_session_version(session, expected_version)\n logger.reason(\"Persisting compiled preview and staling previous preview snapshots\", extra={\"session_id\": session_id, \"user_id\": user_id})\n\n db.query(CompiledPreview).filter(CompiledPreview.session_id == session_id).update({\"preview_status\": \"stale\"})\n db.add(preview)\n db.flush()\n session_record.last_preview_id = preview.preview_id\n\n commit_session_mutation(session, refresh_targets=[preview], expected_version=expected_version)\n logger.reflect(\"Compiled preview committed as latest session preview\", extra={\"session_id\": session.session_id, \"preview_id\": preview.preview_id})\n return preview\n\n\n# [/DEF:save_preview:Function]\n\n\n# [DEF:save_run_context:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist an immutable launch audit snapshot for an owned session.\n# @PRE: session_id belongs to user_id and run_context targets the same aggregate.\n# @POST: run context is persisted and linked as the latest launch snapshot for the session.\n# @SIDE_EFFECT: inserts a run-context row, mutates the parent session pointer, and commits.\ndef save_run_context(\n db: Session,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n session_id: str,\n user_id: str,\n run_context: DatasetRunContext,\n expected_version: Optional[int] = None,\n) -> DatasetRunContext:\n with belief_scope(\"save_run_context\"):\n session = get_owned_session(session_id, user_id)\n session_record = cast(Any, session)\n if expected_version is not None:\n require_session_version(session, expected_version)\n logger.reason(\"Persisting dataset run context audit snapshot\", extra={\"session_id\": session_id, \"user_id\": user_id})\n\n db.add(run_context)\n db.flush()\n session_record.last_run_context_id = run_context.run_context_id\n\n commit_session_mutation(session, refresh_targets=[run_context], expected_version=expected_version)\n logger.reflect(\"Dataset run context committed as latest launch snapshot\", extra={\"session_id\": session.session_id, \"run_context_id\": run_context.run_context_id})\n return run_context\n\n\n# [/DEF:save_run_context:Function]\n\n\n# [/DEF:SessionRepositoryMutations:Module]\n" + "body": "# [DEF:SessionRepositoryMutations:Module]\n# @COMPLEXITY: 4\n# @PURPOSE: Persistence mutation operations for dataset review session aggregates — profile/findings, recovery state, preview, run context.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewModels]\n# @RELATION: DEPENDS_ON -> [SessionEventLogger]\n# @PRE: All mutations execute within authenticated request or task scope.\n# @POST: Session aggregate writes preserve ownership and version semantics.\n\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import Any, List, Optional, cast\n\nfrom sqlalchemy.orm import Session\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.dataset_review import (\n ClarificationQuestion,\n ClarificationSession,\n CompiledPreview,\n DatasetProfile,\n DatasetReviewSession,\n DatasetRunContext,\n ExecutionMapping,\n ImportedFilter,\n SemanticFieldEntry,\n SessionCollaborator,\n SessionEvent,\n TemplateVariable,\n ValidationFinding,\n)\nfrom src.services.dataset_review.event_logger import SessionEventLogger\n\nlog = MarkerLogger(\"SessionMutations\")\n\n\n# [DEF:save_profile_and_findings:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist profile state and replace validation findings for an owned session in one transaction.\n# @PRE: session_id belongs to user_id and the supplied profile/findings belong to the same aggregate scope.\n# @POST: stored profile matches the current session and findings are replaced by the supplied collection.\n# @SIDE_EFFECT: updates profile rows, deletes stale findings, inserts current findings, and commits the transaction.\ndef save_profile_and_findings(\n db: Session,\n event_logger: SessionEventLogger,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n session_id: str,\n user_id: str,\n profile: DatasetProfile,\n findings: List[ValidationFinding],\n expected_version: Optional[int] = None,\n) -> DatasetReviewSession:\n with belief_scope(\"save_profile_and_findings\"):\n session = get_owned_session(session_id, user_id)\n if expected_version is not None:\n require_session_version(session, expected_version)\n log.reason(\"Persisting dataset profile and replacing validation findings\", payload={\"session_id\": session_id, \"user_id\": user_id, \"has_profile\": bool(profile), \"findings_count\": len(findings)})\n\n if profile:\n existing_profile = db.query(DatasetProfile).filter_by(session_id=session_id).first()\n if existing_profile:\n profile.profile_id = existing_profile.profile_id\n db.merge(profile)\n\n db.query(ValidationFinding).filter(ValidationFinding.session_id == session_id).delete()\n for finding in findings:\n cast(Any, finding).session_id = session_id\n db.add(finding)\n\n commit_session_mutation(session, expected_version=expected_version)\n log.reflect(\"Dataset profile and validation findings committed\", payload={\"session_id\": session.session_id, \"user_id\": user_id, \"findings_count\": len(findings)})\n\n from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository\n return session\n\n\n# [/DEF:save_profile_and_findings:Function]\n\n\n# [DEF:save_recovery_state:Function]\n# @COMPLEXITY: 4\n# @PURPOSE: Persist imported filters, template variables, and initial execution mappings for one owned session.\n# @PRE: session_id belongs to user_id.\n# @POST: Recovery state persisted to database.\n# @SIDE_EFFECT: Writes to database.\ndef save_recovery_state(\n db: Session,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n load_session_detail_fn,\n session_id: str,\n user_id: str,\n imported_filters: List[ImportedFilter],\n template_variables: List[TemplateVariable],\n execution_mappings: List[ExecutionMapping],\n expected_version: Optional[int] = None,\n) -> DatasetReviewSession:\n with belief_scope(\"save_recovery_state\"):\n session = get_owned_session(session_id, user_id)\n if expected_version is not None:\n require_session_version(session, expected_version)\n log.reason(\"Persisting dataset review recovery bootstrap state\", payload={\"session_id\": session_id, \"user_id\": user_id, \"imported_filters_count\": len(imported_filters), \"template_variables_count\": len(template_variables), \"execution_mappings_count\": len(execution_mappings)})\n\n db.query(ExecutionMapping).filter(ExecutionMapping.session_id == session_id).delete()\n db.query(TemplateVariable).filter(TemplateVariable.session_id == session_id).delete()\n db.query(ImportedFilter).filter(ImportedFilter.session_id == session_id).delete()\n\n for f in imported_filters:\n cast(Any, f).session_id = session_id\n db.add(f)\n for tv in template_variables:\n cast(Any, tv).session_id = session_id\n db.add(tv)\n db.flush()\n for em in execution_mappings:\n cast(Any, em).session_id = session_id\n db.add(em)\n\n commit_session_mutation(session, expected_version=expected_version)\n log.reflect(\"Dataset review recovery bootstrap state committed\", payload={\"session_id\": session.session_id, \"user_id\": user_id})\n return load_session_detail_fn(session_id, user_id)\n\n\n# [/DEF:save_recovery_state:Function]\n\n\n# [DEF:save_preview:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist a preview snapshot and mark prior session previews stale.\n# @PRE: session_id belongs to user_id and preview is prepared for the same session aggregate.\n# @POST: preview is persisted and the session points to the latest preview identifier.\n# @SIDE_EFFECT: updates prior preview statuses, inserts a preview row, mutates the parent session, and commits.\ndef save_preview(\n db: Session,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n session_id: str,\n user_id: str,\n preview: CompiledPreview,\n expected_version: Optional[int] = None,\n) -> CompiledPreview:\n with belief_scope(\"save_preview\"):\n session = get_owned_session(session_id, user_id)\n session_record = cast(Any, session)\n if expected_version is not None:\n require_session_version(session, expected_version)\n log.reason(\"Persisting compiled preview and staling previous preview snapshots\", payload={\"session_id\": session_id, \"user_id\": user_id})\n\n db.query(CompiledPreview).filter(CompiledPreview.session_id == session_id).update({\"preview_status\": \"stale\"})\n db.add(preview)\n db.flush()\n session_record.last_preview_id = preview.preview_id\n\n commit_session_mutation(session, refresh_targets=[preview], expected_version=expected_version)\n log.reflect(\"Compiled preview committed as latest session preview\", payload={\"session_id\": session.session_id, \"preview_id\": preview.preview_id})\n return preview\n\n\n# [/DEF:save_preview:Function]\n\n\n# [DEF:save_run_context:Function]\n# @COMPLEXITY: 3\n# @PURPOSE: Persist an immutable launch audit snapshot for an owned session.\n# @PRE: session_id belongs to user_id and run_context targets the same aggregate.\n# @POST: run context is persisted and linked as the latest launch snapshot for the session.\n# @SIDE_EFFECT: inserts a run-context row, mutates the parent session pointer, and commits.\ndef save_run_context(\n db: Session,\n get_owned_session,\n require_session_version,\n commit_session_mutation,\n session_id: str,\n user_id: str,\n run_context: DatasetRunContext,\n expected_version: Optional[int] = None,\n) -> DatasetRunContext:\n with belief_scope(\"save_run_context\"):\n session = get_owned_session(session_id, user_id)\n session_record = cast(Any, session)\n if expected_version is not None:\n require_session_version(session, expected_version)\n log.reason(\"Persisting dataset run context audit snapshot\", payload={\"session_id\": session_id, \"user_id\": user_id})\n\n db.add(run_context)\n db.flush()\n session_record.last_run_context_id = run_context.run_context_id\n\n commit_session_mutation(session, refresh_targets=[run_context], expected_version=expected_version)\n log.reflect(\"Dataset run context committed as latest launch snapshot\", payload={\"session_id\": session.session_id, \"run_context_id\": run_context.run_context_id})\n return run_context\n\n\n# [/DEF:save_run_context:Function]\n\n\n# [/DEF:SessionRepositoryMutations:Module]\n" }, { "contract_id": "DatasetReviewSessionRepository", "contract_type": "Module", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", "start_line": 1, - "end_line": 287, + "end_line": 288, "tier": "TIER_3", "complexity": 5, "metadata": { @@ -82642,14 +83651,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:DatasetReviewSessionRepository:Module]\n# @COMPLEXITY: 5\n# @PURPOSE: Persist and retrieve dataset review session aggregates, including readiness, findings, semantic decisions, clarification state, previews, and run contexts.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewSession]\n# @RELATION: DEPENDS_ON -> [DatasetProfile]\n# @RELATION: DEPENDS_ON -> [ValidationFinding]\n# @RELATION: DEPENDS_ON -> [CompiledPreview]\n# @RELATION: DISPATCHES -> [SessionRepositoryMutations:Module]\n# @PRE: repository operations execute within authenticated request or task scope.\n# @POST: session aggregate reads are structurally consistent and writes preserve ownership and version semantics.\n# @SIDE_EFFECT: reads and writes SQLAlchemy-backed session aggregates.\n# @DATA_CONTRACT: Input[SessionMutation] -> Output[PersistedSessionAggregate]\n# @INVARIANT: answers, mapping approvals, preview artifacts, and launch snapshots are never attributed to the wrong user or session.\n# @RATIONALE: Original 627-line file exceeded INV_7 (400-line module limit). Extracted mutation operations into _mutations sub-module.\n# @REJECTED: Keeping all repository operations in one file because it exceeded the fractal limit.\n\nfrom datetime import datetime\nfrom typing import Any, Optional, List, cast\nfrom sqlalchemy import or_\nfrom sqlalchemy.orm import Session, joinedload\nfrom sqlalchemy.orm.exc import StaleDataError\nfrom src.models.dataset_review import (\n ClarificationQuestion,\n ClarificationSession,\n DatasetReviewSession,\n DatasetProfile,\n ValidationFinding,\n CompiledPreview,\n DatasetRunContext,\n ExecutionMapping,\n ImportedFilter,\n SemanticFieldEntry,\n SessionCollaborator,\n SessionEvent,\n TemplateVariable,\n)\nfrom src.core.logger import belief_scope, logger\nfrom src.services.dataset_review.event_logger import SessionEventLogger\n\nlogger = cast(Any, logger)\n\n\n# [DEF:DatasetReviewSessionVersionConflictError:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Signal optimistic-lock conflicts for dataset review session mutations.\nclass DatasetReviewSessionVersionConflictError(ValueError):\n def __init__(self, session_id: str, expected_version: int, actual_version: int):\n self.session_id = session_id\n self.expected_version = expected_version\n self.actual_version = actual_version\n super().__init__(\n f\"Session version conflict: expected {expected_version}, actual {actual_version}\"\n )\n\n\n# [/DEF:DatasetReviewSessionVersionConflictError:Class]\n\n\n# [DEF:DatasetReviewSessionRepository:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Enforce ownership-scoped persistence and retrieval for dataset review session aggregates.\n# @RELATION: DEPENDS_ON -> [DatasetReviewSession]\n# @RELATION: DEPENDS_ON -> [SessionEventLogger]\n# @PRE: constructor receives a live SQLAlchemy session and callers provide authenticated user scope.\n# @POST: repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning.\n# @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session.\nclass DatasetReviewSessionRepository:\n # [DEF:init_repo:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind one live SQLAlchemy session to the repository instance.\n # @PRE: db_session is not None\n # @POST: Repository instance initialized with valid session\n def __init__(self, db: Session):\n self.db = db\n self.event_logger = SessionEventLogger(db)\n\n # [/DEF:init_repo:Function]\n\n # [DEF:get_owned_session:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths.\n # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope.\n # @POST: returns the owned session or raises a deterministic access error.\n def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.get_owned_session\"):\n logger.reason(\"Resolving owner-scoped dataset review session\", extra={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.session_id == session_id, DatasetReviewSession.user_id == user_id)\n .first()\n )\n if not session:\n logger.explore(\"Owner-scoped dataset review session lookup failed\", extra={\"session_id\": session_id, \"user_id\": user_id})\n raise ValueError(\"Session not found or access denied\")\n logger.reflect(\"Owner-scoped dataset review session resolved\", extra={\"session_id\": session.session_id})\n return session\n\n # [/DEF:get_owned_session:Function]\n\n # [DEF:create_sess:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist an initial dataset review session shell.\n # @POST: session is committed, refreshed, and returned with persisted identifiers.\n def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.create_session\"):\n logger.reason(\"Persisting dataset review session shell\", extra={\"user_id\": session.user_id, \"environment_id\": session.environment_id})\n self.db.add(session)\n self.db.commit()\n self.db.refresh(session)\n logger.reflect(\"Dataset review session shell persisted\", extra={\"session_id\": session.session_id})\n return session\n\n # [/DEF:create_sess:Function]\n\n # [DEF:require_session_version:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted.\n # @POST: returns the same session when versions match; otherwise raises deterministic conflict error.\n def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.require_session_version\"):\n actual_version = int(getattr(session, \"version\", 0) or 0)\n logger.reason(\"Checking optimistic-lock version\", extra={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version})\n if actual_version != expected_version:\n logger.explore(\"Rejected mutation due to stale session version\", extra={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version})\n raise DatasetReviewSessionVersionConflictError(str(session.session_id), expected_version, actual_version)\n logger.reflect(\"Optimistic-lock version accepted\", extra={\"session_id\": session.session_id, \"version\": actual_version})\n return session\n\n # [/DEF:require_session_version:Function]\n\n # [DEF:bump_session_version:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled.\n # @POST: session version increments monotonically.\n def bump_session_version(self, session: DatasetReviewSession) -> int:\n with belief_scope(\"DatasetReviewSessionRepository.bump_session_version\"):\n next_version = int(getattr(session, \"version\", 0) or 0) + 1\n setattr(session, \"version\", next_version)\n session.last_activity_at = datetime.utcnow()\n logger.reflect(\"Prepared incremented session version\", extra={\"session_id\": session.session_id, \"version\": next_version})\n return next_version\n\n # [/DEF:bump_session_version:Function]\n\n # [DEF:commit_session_mutation:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts.\n # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised.\n def commit_session_mutation(\n self, session: DatasetReviewSession, *, refresh_targets: Optional[List[Any]] = None, expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.commit_session_mutation\"):\n observed_version = int(expected_version if expected_version is not None else getattr(session, \"version\", 0) or 0)\n logger.reason(\"Committing session mutation with optimistic lock\", extra={\"session_id\": session.session_id, \"observed_version\": observed_version})\n self.bump_session_version(session)\n try:\n self.db.commit()\n except StaleDataError as exc:\n self.db.rollback()\n actual_version_row = self.db.query(DatasetReviewSession.version).filter(DatasetReviewSession.session_id == session.session_id).first()\n actual_version = int(actual_version_row[0] or 0) if actual_version_row else 0\n logger.explore(\"Session commit rejected by optimistic lock\", extra={\"session_id\": session.session_id, \"expected_version\": observed_version, \"actual_version\": actual_version})\n raise DatasetReviewSessionVersionConflictError(session.session_id, observed_version, actual_version) from exc\n self.db.refresh(session)\n for target in refresh_targets or []:\n self.db.refresh(target)\n logger.reflect(\"Session mutation committed\", extra={\"session_id\": session.session_id, \"version\": getattr(session, \"version\", None)})\n return session\n\n # [/DEF:commit_session_mutation:Function]\n\n # [DEF:load_detail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return the full session aggregate for API and frontend resume flows.\n # @POST: Returns SessionDetail with all fields populated or None.\n def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.load_session_detail\"):\n logger.reason(\"Loading dataset review session detail\", extra={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id)\n .options(\n joinedload(DatasetReviewSession.profile),\n joinedload(DatasetReviewSession.findings),\n joinedload(DatasetReviewSession.collaborators),\n joinedload(DatasetReviewSession.semantic_sources),\n joinedload(DatasetReviewSession.semantic_fields).joinedload(SemanticFieldEntry.candidates),\n joinedload(DatasetReviewSession.imported_filters),\n joinedload(DatasetReviewSession.template_variables),\n joinedload(DatasetReviewSession.execution_mappings),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.options),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.answer),\n joinedload(DatasetReviewSession.previews),\n joinedload(DatasetReviewSession.run_contexts),\n joinedload(DatasetReviewSession.events),\n )\n .filter(DatasetReviewSession.session_id == session_id)\n .filter(or_(DatasetReviewSession.user_id == user_id, SessionCollaborator.user_id == user_id))\n .first()\n )\n logger.reflect(\"Session detail lookup completed\", extra={\"session_id\": session_id, \"found\": bool(session)})\n return session\n\n # [/DEF:load_detail:Function]\n\n # [DEF:save_profile_and_findings:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist profile state and replace validation findings for an owned session.\n # @POST: stored profile matches the current session and findings are replaced.\n def save_profile_and_findings(\n self, session_id: str, user_id: str, profile: DatasetProfile, findings: List[ValidationFinding], expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_profile_and_findings as _save\n return _save(\n self.db, self.event_logger, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, session_id, user_id, profile, findings, expected_version,\n )\n\n # [/DEF:save_profile_and_findings:Function]\n\n # [DEF:save_recovery_state:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist imported filters, template variables, and initial execution mappings.\n def save_recovery_state(\n self, session_id: str, user_id: str, imported_filters: List[ImportedFilter],\n template_variables: List[TemplateVariable], execution_mappings: List[ExecutionMapping],\n expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_recovery_state as _save\n return _save(\n self.db, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, self.load_session_detail,\n session_id, user_id, imported_filters, template_variables, execution_mappings, expected_version,\n )\n\n # [/DEF:save_recovery_state:Function]\n\n # [DEF:save_preview:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist a preview snapshot and mark prior session previews stale.\n def save_preview(\n self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: Optional[int] = None,\n ) -> CompiledPreview:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_preview as _save\n return _save(\n self.db, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, session_id, user_id, preview, expected_version,\n )\n\n # [/DEF:save_preview:Function]\n\n # [DEF:save_run_context:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist an immutable launch audit snapshot for an owned session.\n def save_run_context(\n self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: Optional[int] = None,\n ) -> DatasetRunContext:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_run_context as _save\n return _save(\n self.db, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, session_id, user_id, run_context, expected_version,\n )\n\n # [/DEF:save_run_context:Function]\n\n # [DEF:list_user_sess:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: List review sessions owned by a specific user ordered by most recent update.\n def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.list_sessions_for_user\"):\n logger.reason(\"Listing dataset review sessions for owner scope\", extra={\"user_id\": user_id})\n sessions = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.user_id == user_id)\n .order_by(DatasetReviewSession.updated_at.desc())\n .all()\n )\n logger.reflect(\"Session list assembled\", extra={\"user_id\": user_id, \"session_count\": len(sessions)})\n return sessions\n\n # [/DEF:list_user_sess:Function]\n\n\n# [/DEF:DatasetReviewSessionRepository:Class]\n\n# [/DEF:DatasetReviewSessionRepository:Module]\n" + "body": "# [DEF:DatasetReviewSessionRepository:Module]\n# @COMPLEXITY: 5\n# @PURPOSE: Persist and retrieve dataset review session aggregates, including readiness, findings, semantic decisions, clarification state, previews, and run contexts.\n# @LAYER: Domain\n# @RELATION: DEPENDS_ON -> [DatasetReviewSession]\n# @RELATION: DEPENDS_ON -> [DatasetProfile]\n# @RELATION: DEPENDS_ON -> [ValidationFinding]\n# @RELATION: DEPENDS_ON -> [CompiledPreview]\n# @RELATION: DISPATCHES -> [SessionRepositoryMutations:Module]\n# @PRE: repository operations execute within authenticated request or task scope.\n# @POST: session aggregate reads are structurally consistent and writes preserve ownership and version semantics.\n# @SIDE_EFFECT: reads and writes SQLAlchemy-backed session aggregates.\n# @DATA_CONTRACT: Input[SessionMutation] -> Output[PersistedSessionAggregate]\n# @INVARIANT: answers, mapping approvals, preview artifacts, and launch snapshots are never attributed to the wrong user or session.\n# @RATIONALE: Original 627-line file exceeded INV_7 (400-line module limit). Extracted mutation operations into _mutations sub-module.\n# @REJECTED: Keeping all repository operations in one file because it exceeded the fractal limit.\n\nfrom datetime import datetime\nfrom typing import Any, Optional, List\nfrom sqlalchemy import or_\nfrom sqlalchemy.orm import Session, joinedload\nfrom sqlalchemy.orm.exc import StaleDataError\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.dataset_review import (\n ClarificationQuestion,\n ClarificationSession,\n DatasetReviewSession,\n DatasetProfile,\n ValidationFinding,\n CompiledPreview,\n DatasetRunContext,\n ExecutionMapping,\n ImportedFilter,\n SemanticFieldEntry,\n SessionCollaborator,\n SessionEvent,\n TemplateVariable,\n)\nfrom src.services.dataset_review.event_logger import SessionEventLogger\n\nlog = MarkerLogger(\"SessionRepository\")\n\n\n# [DEF:DatasetReviewSessionVersionConflictError:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Signal optimistic-lock conflicts for dataset review session mutations.\nclass DatasetReviewSessionVersionConflictError(ValueError):\n def __init__(self, session_id: str, expected_version: int, actual_version: int):\n self.session_id = session_id\n self.expected_version = expected_version\n self.actual_version = actual_version\n super().__init__(\n f\"Session version conflict: expected {expected_version}, actual {actual_version}\"\n )\n\n\n# [/DEF:DatasetReviewSessionVersionConflictError:Class]\n\n\n# [DEF:DatasetReviewSessionRepository:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Enforce ownership-scoped persistence and retrieval for dataset review session aggregates.\n# @RELATION: DEPENDS_ON -> [DatasetReviewSession]\n# @RELATION: DEPENDS_ON -> [SessionEventLogger]\n# @PRE: constructor receives a live SQLAlchemy session and callers provide authenticated user scope.\n# @POST: repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning.\n# @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session.\nclass DatasetReviewSessionRepository:\n # [DEF:init_repo:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Bind one live SQLAlchemy session to the repository instance.\n # @PRE: db_session is not None\n # @POST: Repository instance initialized with valid session\n def __init__(self, db: Session):\n self.db = db\n self.event_logger = SessionEventLogger(db)\n\n # [/DEF:init_repo:Function]\n\n # [DEF:get_owned_session:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths.\n # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope.\n # @POST: returns the owned session or raises a deterministic access error.\n def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.get_owned_session\"):\n log.reason(\"Resolving owner-scoped dataset review session\", payload={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.session_id == session_id, DatasetReviewSession.user_id == user_id)\n .first()\n )\n if not session:\n log.explore(\"Owner-scoped dataset review session lookup failed\", payload={\"session_id\": session_id, \"user_id\": user_id}, error=\"Session not found or access denied\")\n raise ValueError(\"Session not found or access denied\")\n log.reflect(\"Owner-scoped dataset review session resolved\", payload={\"session_id\": session.session_id})\n return session\n\n # [/DEF:get_owned_session:Function]\n\n # [DEF:create_sess:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist an initial dataset review session shell.\n # @POST: session is committed, refreshed, and returned with persisted identifiers.\n def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.create_session\"):\n log.reason(\"Persisting dataset review session shell\", payload={\"user_id\": session.user_id, \"environment_id\": session.environment_id})\n self.db.add(session)\n self.db.commit()\n self.db.refresh(session)\n log.reflect(\"Dataset review session shell persisted\", payload={\"session_id\": session.session_id})\n return session\n\n # [/DEF:create_sess:Function]\n\n # [DEF:require_session_version:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted.\n # @POST: returns the same session when versions match; otherwise raises deterministic conflict error.\n def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.require_session_version\"):\n actual_version = int(getattr(session, \"version\", 0) or 0)\n log.reason(\"Checking optimistic-lock version\", payload={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version})\n if actual_version != expected_version:\n log.explore(\"Rejected mutation due to stale session version\", payload={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version}, error=\"Optimistic lock version mismatch\")\n raise DatasetReviewSessionVersionConflictError(str(session.session_id), expected_version, actual_version)\n log.reflect(\"Optimistic-lock version accepted\", payload={\"session_id\": session.session_id, \"version\": actual_version})\n return session\n\n # [/DEF:require_session_version:Function]\n\n # [DEF:bump_session_version:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled.\n # @POST: session version increments monotonically.\n def bump_session_version(self, session: DatasetReviewSession) -> int:\n with belief_scope(\"DatasetReviewSessionRepository.bump_session_version\"):\n next_version = int(getattr(session, \"version\", 0) or 0) + 1\n setattr(session, \"version\", next_version)\n session.last_activity_at = datetime.utcnow()\n log.reflect(\"Prepared incremented session version\", payload={\"session_id\": session.session_id, \"version\": next_version})\n return next_version\n\n # [/DEF:bump_session_version:Function]\n\n # [DEF:commit_session_mutation:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts.\n # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised.\n def commit_session_mutation(\n self, session: DatasetReviewSession, *, refresh_targets: Optional[List[Any]] = None, expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.commit_session_mutation\"):\n observed_version = int(expected_version if expected_version is not None else getattr(session, \"version\", 0) or 0)\n log.reason(\"Committing session mutation with optimistic lock\", payload={\"session_id\": session.session_id, \"observed_version\": observed_version})\n self.bump_session_version(session)\n try:\n self.db.commit()\n except StaleDataError as exc:\n self.db.rollback()\n actual_version_row = self.db.query(DatasetReviewSession.version).filter(DatasetReviewSession.session_id == session.session_id).first()\n actual_version = int(actual_version_row[0] or 0) if actual_version_row else 0\n log.explore(\"Session commit rejected by optimistic lock\", payload={\"session_id\": session.session_id, \"expected_version\": observed_version, \"actual_version\": actual_version}, error=\"StaleDataError on session commit\")\n raise DatasetReviewSessionVersionConflictError(session.session_id, observed_version, actual_version) from exc\n self.db.refresh(session)\n for target in refresh_targets or []:\n self.db.refresh(target)\n log.reflect(\"Session mutation committed\", payload={\"session_id\": session.session_id, \"version\": getattr(session, \"version\", None)})\n return session\n\n # [/DEF:commit_session_mutation:Function]\n\n # [DEF:load_detail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return the full session aggregate for API and frontend resume flows.\n # @POST: Returns SessionDetail with all fields populated or None.\n def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.load_session_detail\"):\n log.reason(\"Loading dataset review session detail\", payload={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id)\n .options(\n joinedload(DatasetReviewSession.profile),\n joinedload(DatasetReviewSession.findings),\n joinedload(DatasetReviewSession.collaborators),\n joinedload(DatasetReviewSession.semantic_sources),\n joinedload(DatasetReviewSession.semantic_fields).joinedload(SemanticFieldEntry.candidates),\n joinedload(DatasetReviewSession.imported_filters),\n joinedload(DatasetReviewSession.template_variables),\n joinedload(DatasetReviewSession.execution_mappings),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.options),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.answer),\n joinedload(DatasetReviewSession.previews),\n joinedload(DatasetReviewSession.run_contexts),\n joinedload(DatasetReviewSession.events),\n )\n .filter(DatasetReviewSession.session_id == session_id)\n .filter(or_(DatasetReviewSession.user_id == user_id, SessionCollaborator.user_id == user_id))\n .first()\n )\n log.reflect(\"Session detail lookup completed\", payload={\"session_id\": session_id, \"found\": bool(session)})\n return session\n\n # [/DEF:load_detail:Function]\n\n # [DEF:save_profile_and_findings:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Persist profile state and replace validation findings for an owned session.\n # @POST: stored profile matches the current session and findings are replaced.\n def save_profile_and_findings(\n self, session_id: str, user_id: str, profile: DatasetProfile, findings: List[ValidationFinding], expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_profile_and_findings as _save\n return _save(\n self.db, self.event_logger, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, session_id, user_id, profile, findings, expected_version,\n )\n\n # [/DEF:save_profile_and_findings:Function]\n\n # [DEF:save_recovery_state:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist imported filters, template variables, and initial execution mappings.\n def save_recovery_state(\n self, session_id: str, user_id: str, imported_filters: List[ImportedFilter],\n template_variables: List[TemplateVariable], execution_mappings: List[ExecutionMapping],\n expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_recovery_state as _save\n return _save(\n self.db, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, self.load_session_detail,\n session_id, user_id, imported_filters, template_variables, execution_mappings, expected_version,\n )\n\n # [/DEF:save_recovery_state:Function]\n\n # [DEF:save_preview:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist a preview snapshot and mark prior session previews stale.\n def save_preview(\n self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: Optional[int] = None,\n ) -> CompiledPreview:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_preview as _save\n return _save(\n self.db, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, session_id, user_id, preview, expected_version,\n )\n\n # [/DEF:save_preview:Function]\n\n # [DEF:save_run_context:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist an immutable launch audit snapshot for an owned session.\n def save_run_context(\n self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: Optional[int] = None,\n ) -> DatasetRunContext:\n from src.services.dataset_review.repositories.repository_pkg._mutations import save_run_context as _save\n return _save(\n self.db, self._get_owned_session, self.require_session_version,\n self.commit_session_mutation, session_id, user_id, run_context, expected_version,\n )\n\n # [/DEF:save_run_context:Function]\n\n # [DEF:list_user_sess:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: List review sessions owned by a specific user ordered by most recent update.\n def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.list_sessions_for_user\"):\n log.reason(\"Listing dataset review sessions for owner scope\", payload={\"user_id\": user_id})\n sessions = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.user_id == user_id)\n .order_by(DatasetReviewSession.updated_at.desc())\n .all()\n )\n log.reflect(\"Session list assembled\", payload={\"user_id\": user_id, \"session_count\": len(sessions)})\n return sessions\n\n # [/DEF:list_user_sess:Function]\n\n\n# [/DEF:DatasetReviewSessionRepository:Class]\n\n# [/DEF:DatasetReviewSessionRepository:Module]\n" }, { "contract_id": "DatasetReviewSessionVersionConflictError", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 44, - "end_line": 57, + "start_line": 45, + "end_line": 58, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -82665,8 +83674,8 @@ "contract_id": "init_repo", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 69, - "end_line": 78, + "start_line": 70, + "end_line": 79, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -82703,8 +83712,8 @@ "contract_id": "get_owned_session", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 80, - "end_line": 99, + "start_line": 81, + "end_line": 100, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -82744,14 +83753,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_owned_session:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths.\n # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope.\n # @POST: returns the owned session or raises a deterministic access error.\n def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.get_owned_session\"):\n logger.reason(\"Resolving owner-scoped dataset review session\", extra={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.session_id == session_id, DatasetReviewSession.user_id == user_id)\n .first()\n )\n if not session:\n logger.explore(\"Owner-scoped dataset review session lookup failed\", extra={\"session_id\": session_id, \"user_id\": user_id})\n raise ValueError(\"Session not found or access denied\")\n logger.reflect(\"Owner-scoped dataset review session resolved\", extra={\"session_id\": session.session_id})\n return session\n\n # [/DEF:get_owned_session:Function]\n" + "body": " # [DEF:get_owned_session:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths.\n # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope.\n # @POST: returns the owned session or raises a deterministic access error.\n def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.get_owned_session\"):\n log.reason(\"Resolving owner-scoped dataset review session\", payload={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.session_id == session_id, DatasetReviewSession.user_id == user_id)\n .first()\n )\n if not session:\n log.explore(\"Owner-scoped dataset review session lookup failed\", payload={\"session_id\": session_id, \"user_id\": user_id}, error=\"Session not found or access denied\")\n raise ValueError(\"Session not found or access denied\")\n log.reflect(\"Owner-scoped dataset review session resolved\", payload={\"session_id\": session.session_id})\n return session\n\n # [/DEF:get_owned_session:Function]\n" }, { "contract_id": "create_sess", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 101, - "end_line": 114, + "start_line": 102, + "end_line": 115, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -82781,14 +83790,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:create_sess:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist an initial dataset review session shell.\n # @POST: session is committed, refreshed, and returned with persisted identifiers.\n def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.create_session\"):\n logger.reason(\"Persisting dataset review session shell\", extra={\"user_id\": session.user_id, \"environment_id\": session.environment_id})\n self.db.add(session)\n self.db.commit()\n self.db.refresh(session)\n logger.reflect(\"Dataset review session shell persisted\", extra={\"session_id\": session.session_id})\n return session\n\n # [/DEF:create_sess:Function]\n" + "body": " # [DEF:create_sess:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Persist an initial dataset review session shell.\n # @POST: session is committed, refreshed, and returned with persisted identifiers.\n def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.create_session\"):\n log.reason(\"Persisting dataset review session shell\", payload={\"user_id\": session.user_id, \"environment_id\": session.environment_id})\n self.db.add(session)\n self.db.commit()\n self.db.refresh(session)\n log.reflect(\"Dataset review session shell persisted\", payload={\"session_id\": session.session_id})\n return session\n\n # [/DEF:create_sess:Function]\n" }, { "contract_id": "require_session_version", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 116, - "end_line": 130, + "start_line": 117, + "end_line": 131, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -82818,14 +83827,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:require_session_version:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted.\n # @POST: returns the same session when versions match; otherwise raises deterministic conflict error.\n def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.require_session_version\"):\n actual_version = int(getattr(session, \"version\", 0) or 0)\n logger.reason(\"Checking optimistic-lock version\", extra={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version})\n if actual_version != expected_version:\n logger.explore(\"Rejected mutation due to stale session version\", extra={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version})\n raise DatasetReviewSessionVersionConflictError(str(session.session_id), expected_version, actual_version)\n logger.reflect(\"Optimistic-lock version accepted\", extra={\"session_id\": session.session_id, \"version\": actual_version})\n return session\n\n # [/DEF:require_session_version:Function]\n" + "body": " # [DEF:require_session_version:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted.\n # @POST: returns the same session when versions match; otherwise raises deterministic conflict error.\n def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.require_session_version\"):\n actual_version = int(getattr(session, \"version\", 0) or 0)\n log.reason(\"Checking optimistic-lock version\", payload={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version})\n if actual_version != expected_version:\n log.explore(\"Rejected mutation due to stale session version\", payload={\"session_id\": session.session_id, \"expected_version\": expected_version, \"actual_version\": actual_version}, error=\"Optimistic lock version mismatch\")\n raise DatasetReviewSessionVersionConflictError(str(session.session_id), expected_version, actual_version)\n log.reflect(\"Optimistic-lock version accepted\", payload={\"session_id\": session.session_id, \"version\": actual_version})\n return session\n\n # [/DEF:require_session_version:Function]\n" }, { "contract_id": "bump_session_version", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 132, - "end_line": 144, + "start_line": 133, + "end_line": 145, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -82846,14 +83855,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:bump_session_version:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled.\n # @POST: session version increments monotonically.\n def bump_session_version(self, session: DatasetReviewSession) -> int:\n with belief_scope(\"DatasetReviewSessionRepository.bump_session_version\"):\n next_version = int(getattr(session, \"version\", 0) or 0) + 1\n setattr(session, \"version\", next_version)\n session.last_activity_at = datetime.utcnow()\n logger.reflect(\"Prepared incremented session version\", extra={\"session_id\": session.session_id, \"version\": next_version})\n return next_version\n\n # [/DEF:bump_session_version:Function]\n" + "body": " # [DEF:bump_session_version:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled.\n # @POST: session version increments monotonically.\n def bump_session_version(self, session: DatasetReviewSession) -> int:\n with belief_scope(\"DatasetReviewSessionRepository.bump_session_version\"):\n next_version = int(getattr(session, \"version\", 0) or 0) + 1\n setattr(session, \"version\", next_version)\n session.last_activity_at = datetime.utcnow()\n log.reflect(\"Prepared incremented session version\", payload={\"session_id\": session.session_id, \"version\": next_version})\n return next_version\n\n # [/DEF:bump_session_version:Function]\n" }, { "contract_id": "commit_session_mutation", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 146, - "end_line": 171, + "start_line": 147, + "end_line": 172, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -82892,14 +83901,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:commit_session_mutation:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts.\n # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised.\n def commit_session_mutation(\n self, session: DatasetReviewSession, *, refresh_targets: Optional[List[Any]] = None, expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.commit_session_mutation\"):\n observed_version = int(expected_version if expected_version is not None else getattr(session, \"version\", 0) or 0)\n logger.reason(\"Committing session mutation with optimistic lock\", extra={\"session_id\": session.session_id, \"observed_version\": observed_version})\n self.bump_session_version(session)\n try:\n self.db.commit()\n except StaleDataError as exc:\n self.db.rollback()\n actual_version_row = self.db.query(DatasetReviewSession.version).filter(DatasetReviewSession.session_id == session.session_id).first()\n actual_version = int(actual_version_row[0] or 0) if actual_version_row else 0\n logger.explore(\"Session commit rejected by optimistic lock\", extra={\"session_id\": session.session_id, \"expected_version\": observed_version, \"actual_version\": actual_version})\n raise DatasetReviewSessionVersionConflictError(session.session_id, observed_version, actual_version) from exc\n self.db.refresh(session)\n for target in refresh_targets or []:\n self.db.refresh(target)\n logger.reflect(\"Session mutation committed\", extra={\"session_id\": session.session_id, \"version\": getattr(session, \"version\", None)})\n return session\n\n # [/DEF:commit_session_mutation:Function]\n" + "body": " # [DEF:commit_session_mutation:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts.\n # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised.\n def commit_session_mutation(\n self, session: DatasetReviewSession, *, refresh_targets: Optional[List[Any]] = None, expected_version: Optional[int] = None,\n ) -> DatasetReviewSession:\n with belief_scope(\"DatasetReviewSessionRepository.commit_session_mutation\"):\n observed_version = int(expected_version if expected_version is not None else getattr(session, \"version\", 0) or 0)\n log.reason(\"Committing session mutation with optimistic lock\", payload={\"session_id\": session.session_id, \"observed_version\": observed_version})\n self.bump_session_version(session)\n try:\n self.db.commit()\n except StaleDataError as exc:\n self.db.rollback()\n actual_version_row = self.db.query(DatasetReviewSession.version).filter(DatasetReviewSession.session_id == session.session_id).first()\n actual_version = int(actual_version_row[0] or 0) if actual_version_row else 0\n log.explore(\"Session commit rejected by optimistic lock\", payload={\"session_id\": session.session_id, \"expected_version\": observed_version, \"actual_version\": actual_version}, error=\"StaleDataError on session commit\")\n raise DatasetReviewSessionVersionConflictError(session.session_id, observed_version, actual_version) from exc\n self.db.refresh(session)\n for target in refresh_targets or []:\n self.db.refresh(target)\n log.reflect(\"Session mutation committed\", payload={\"session_id\": session.session_id, \"version\": getattr(session, \"version\", None)})\n return session\n\n # [/DEF:commit_session_mutation:Function]\n" }, { "contract_id": "load_detail", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 173, - "end_line": 205, + "start_line": 174, + "end_line": 206, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -82929,14 +83938,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:load_detail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return the full session aggregate for API and frontend resume flows.\n # @POST: Returns SessionDetail with all fields populated or None.\n def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.load_session_detail\"):\n logger.reason(\"Loading dataset review session detail\", extra={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id)\n .options(\n joinedload(DatasetReviewSession.profile),\n joinedload(DatasetReviewSession.findings),\n joinedload(DatasetReviewSession.collaborators),\n joinedload(DatasetReviewSession.semantic_sources),\n joinedload(DatasetReviewSession.semantic_fields).joinedload(SemanticFieldEntry.candidates),\n joinedload(DatasetReviewSession.imported_filters),\n joinedload(DatasetReviewSession.template_variables),\n joinedload(DatasetReviewSession.execution_mappings),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.options),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.answer),\n joinedload(DatasetReviewSession.previews),\n joinedload(DatasetReviewSession.run_contexts),\n joinedload(DatasetReviewSession.events),\n )\n .filter(DatasetReviewSession.session_id == session_id)\n .filter(or_(DatasetReviewSession.user_id == user_id, SessionCollaborator.user_id == user_id))\n .first()\n )\n logger.reflect(\"Session detail lookup completed\", extra={\"session_id\": session_id, \"found\": bool(session)})\n return session\n\n # [/DEF:load_detail:Function]\n" + "body": " # [DEF:load_detail:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Return the full session aggregate for API and frontend resume flows.\n # @POST: Returns SessionDetail with all fields populated or None.\n def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.load_session_detail\"):\n log.reason(\"Loading dataset review session detail\", payload={\"session_id\": session_id, \"user_id\": user_id})\n session = (\n self.db.query(DatasetReviewSession)\n .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id)\n .options(\n joinedload(DatasetReviewSession.profile),\n joinedload(DatasetReviewSession.findings),\n joinedload(DatasetReviewSession.collaborators),\n joinedload(DatasetReviewSession.semantic_sources),\n joinedload(DatasetReviewSession.semantic_fields).joinedload(SemanticFieldEntry.candidates),\n joinedload(DatasetReviewSession.imported_filters),\n joinedload(DatasetReviewSession.template_variables),\n joinedload(DatasetReviewSession.execution_mappings),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.options),\n joinedload(DatasetReviewSession.clarification_sessions).joinedload(ClarificationSession.questions).joinedload(ClarificationQuestion.answer),\n joinedload(DatasetReviewSession.previews),\n joinedload(DatasetReviewSession.run_contexts),\n joinedload(DatasetReviewSession.events),\n )\n .filter(DatasetReviewSession.session_id == session_id)\n .filter(or_(DatasetReviewSession.user_id == user_id, SessionCollaborator.user_id == user_id))\n .first()\n )\n log.reflect(\"Session detail lookup completed\", payload={\"session_id\": session_id, \"found\": bool(session)})\n return session\n\n # [/DEF:load_detail:Function]\n" }, { "contract_id": "save_profile_and_findings", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 207, - "end_line": 220, + "start_line": 208, + "end_line": 221, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -82981,8 +83990,8 @@ "contract_id": "save_recovery_state", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 222, - "end_line": 237, + "start_line": 223, + "end_line": 238, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -83008,8 +84017,8 @@ "contract_id": "save_preview", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 239, - "end_line": 251, + "start_line": 240, + "end_line": 252, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -83035,8 +84044,8 @@ "contract_id": "save_run_context", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 253, - "end_line": 265, + "start_line": 254, + "end_line": 266, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -83062,8 +84071,8 @@ "contract_id": "list_user_sess", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/repositories/session_repository.py", - "start_line": 267, - "end_line": 282, + "start_line": 268, + "end_line": 283, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83073,14 +84082,14 @@ "relations": [], "schema_warnings": [], "anchor_syntax": "def", - "body": " # [DEF:list_user_sess:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: List review sessions owned by a specific user ordered by most recent update.\n def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.list_sessions_for_user\"):\n logger.reason(\"Listing dataset review sessions for owner scope\", extra={\"user_id\": user_id})\n sessions = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.user_id == user_id)\n .order_by(DatasetReviewSession.updated_at.desc())\n .all()\n )\n logger.reflect(\"Session list assembled\", extra={\"user_id\": user_id, \"session_count\": len(sessions)})\n return sessions\n\n # [/DEF:list_user_sess:Function]\n" + "body": " # [DEF:list_user_sess:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: List review sessions owned by a specific user ordered by most recent update.\n def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]:\n with belief_scope(\"DatasetReviewSessionRepository.list_sessions_for_user\"):\n log.reason(\"Listing dataset review sessions for owner scope\", payload={\"user_id\": user_id})\n sessions = (\n self.db.query(DatasetReviewSession)\n .filter(DatasetReviewSession.user_id == user_id)\n .order_by(DatasetReviewSession.updated_at.desc())\n .all()\n )\n log.reflect(\"Session list assembled\", payload={\"user_id\": user_id, \"session_count\": len(sessions)})\n return sessions\n\n # [/DEF:list_user_sess:Function]\n" }, { "contract_id": "SemanticSourceResolver", "contract_type": "Module", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", "start_line": 1, - "end_line": 400, + "end_line": 394, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -83205,14 +84214,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:SemanticSourceResolver:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: dataset_review, semantic_resolution, dictionary, trusted_sources, ranking\n# @PURPOSE: Resolve and rank semantic candidates from trusted dictionary-like sources before any inferred fallback.\n# @LAYER: Domain\n# @RELATION: [DEPENDS_ON] ->[LLMProviderService]\n# @RELATION: [DEPENDS_ON] ->[SemanticSource]\n# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n# @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n# @PRE: selected source and target field set must be known.\n# @POST: candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable.\n# @SIDE_EFFECT: may create conflict findings and semantic candidate records.\n# @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values.\n\nfrom __future__ import annotations\n\n# [DEF:imports:Block]\nfrom dataclasses import dataclass, field\nfrom difflib import SequenceMatcher\nfrom typing import Any, Dict, Iterable, List, Mapping, Optional\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.dataset_review import (\n CandidateMatchType,\n CandidateStatus,\n FieldProvenance,\n SemanticSource,\n)\n# [/DEF:imports:Block]\n\n\n# [DEF:DictionaryResolutionResult:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Carries field-level dictionary resolution output with explicit review and partial-recovery state.\n@dataclass\nclass DictionaryResolutionResult:\n source_ref: str\n resolved_fields: List[Dict[str, Any]] = field(default_factory=list)\n unresolved_fields: List[str] = field(default_factory=list)\n partial_recovery: bool = False\n# [/DEF:DictionaryResolutionResult:Class]\n\n\n# [DEF:SemanticSourceResolver:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Resolve semantic candidates from trusted sources while preserving manual locks and confidence ordering.\n# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n# @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n# @PRE: source payload and target field collection are provided by the caller.\n# @POST: result contains confidence-ranked candidates and does not overwrite manual locks implicitly.\n# @SIDE_EFFECT: emits semantic trace logs for ranking and fallback decisions.\nclass SemanticSourceResolver:\n # [DEF:resolve_from_file:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize uploaded semantic file records into field-level candidates.\n def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult:\n return DictionaryResolutionResult(source_ref=str(source_payload.get(\"source_ref\") or \"uploaded_file\"))\n # [/DEF:resolve_from_file:Function]\n\n # [DEF:resolve_from_dictionary:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Resolve candidates from connected tabular dictionary sources.\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n # @PRE: dictionary source exists and fields contain stable field_name values.\n # @POST: returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit.\n # @SIDE_EFFECT: emits belief-state logs describing trusted-match and partial-recovery outcomes.\n # @DATA_CONTRACT: Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult]\n def resolve_from_dictionary(\n self,\n source_payload: Mapping[str, Any],\n fields: Iterable[Mapping[str, Any]],\n ) -> DictionaryResolutionResult:\n with belief_scope(\"SemanticSourceResolver.resolve_from_dictionary\"):\n source_ref = str(source_payload.get(\"source_ref\") or \"\").strip()\n dictionary_rows = source_payload.get(\"rows\")\n\n if not source_ref:\n logger.explore(\"Dictionary semantic source is missing source_ref\")\n raise ValueError(\"Dictionary semantic source must include source_ref\")\n\n if not isinstance(dictionary_rows, list) or not dictionary_rows:\n logger.explore(\n \"Dictionary semantic source has no usable rows\",\n extra={\"source_ref\": source_ref},\n )\n raise ValueError(\"Dictionary semantic source must include non-empty rows\")\n\n logger.reason(\n \"Resolving semantics from trusted dictionary source\",\n extra={\"source_ref\": source_ref, \"row_count\": len(dictionary_rows)},\n )\n\n normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)]\n row_index = {\n row[\"field_key\"]: row\n for row in normalized_rows\n if row.get(\"field_key\")\n }\n\n resolved_fields: List[Dict[str, Any]] = []\n unresolved_fields: List[str] = []\n\n for raw_field in fields:\n field_name = str(raw_field.get(\"field_name\") or \"\").strip()\n if not field_name:\n continue\n\n is_locked = bool(raw_field.get(\"is_locked\"))\n if is_locked:\n logger.reason(\n \"Preserving manual lock during dictionary resolution\",\n extra={\"field_name\": field_name},\n )\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.MANUAL_OVERRIDE.value,\n \"needs_review\": False,\n \"has_conflict\": False,\n \"is_locked\": True,\n \"status\": \"preserved_manual\",\n }\n )\n continue\n\n exact_match = row_index.get(self._normalize_key(field_name))\n candidates: List[Dict[str, Any]] = []\n\n if exact_match is not None:\n logger.reason(\n \"Resolved exact dictionary match\",\n extra={\"field_name\": field_name, \"source_ref\": source_ref},\n )\n candidates.append(\n self._build_candidate_payload(\n rank=1,\n match_type=CandidateMatchType.EXACT,\n confidence_score=1.0,\n row=exact_match,\n )\n )\n else:\n fuzzy_matches = self._find_fuzzy_matches(field_name, normalized_rows)\n for rank_offset, fuzzy_match in enumerate(fuzzy_matches, start=1):\n candidates.append(\n self._build_candidate_payload(\n rank=rank_offset,\n match_type=CandidateMatchType.FUZZY,\n confidence_score=float(fuzzy_match[\"score\"]),\n row=fuzzy_match[\"row\"],\n )\n )\n\n if not candidates:\n unresolved_fields.append(field_name)\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.UNRESOLVED.value,\n \"needs_review\": True,\n \"has_conflict\": False,\n \"is_locked\": False,\n \"status\": \"unresolved\",\n }\n )\n logger.explore(\n \"No trusted dictionary match found for field\",\n extra={\"field_name\": field_name, \"source_ref\": source_ref},\n )\n continue\n\n ranked_candidates = self.rank_candidates(candidates)\n applied_candidate = ranked_candidates[0]\n has_conflict = len(ranked_candidates) > 1\n provenance = (\n FieldProvenance.DICTIONARY_EXACT.value\n if applied_candidate[\"match_type\"] == CandidateMatchType.EXACT.value\n else FieldProvenance.FUZZY_INFERRED.value\n )\n needs_review = applied_candidate[\"match_type\"] != CandidateMatchType.EXACT.value\n\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": applied_candidate,\n \"candidates\": ranked_candidates,\n \"provenance\": provenance,\n \"needs_review\": needs_review,\n \"has_conflict\": has_conflict,\n \"is_locked\": False,\n \"status\": \"resolved\",\n }\n )\n\n result = DictionaryResolutionResult(\n source_ref=source_ref,\n resolved_fields=resolved_fields,\n unresolved_fields=unresolved_fields,\n partial_recovery=bool(unresolved_fields),\n )\n logger.reflect(\n \"Dictionary resolution completed\",\n extra={\n \"source_ref\": source_ref,\n \"resolved_fields\": len(resolved_fields),\n \"unresolved_fields\": len(unresolved_fields),\n \"partial_recovery\": result.partial_recovery,\n },\n )\n return result\n # [/DEF:resolve_from_dictionary:Function]\n\n # [DEF:resolve_from_reference_dataset:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Reuse semantic metadata from trusted Superset datasets.\n def resolve_from_reference_dataset(\n self,\n source_payload: Mapping[str, Any],\n fields: Iterable[Mapping[str, Any]],\n ) -> DictionaryResolutionResult:\n return DictionaryResolutionResult(source_ref=str(source_payload.get(\"source_ref\") or \"reference_dataset\"))\n # [/DEF:resolve_from_reference_dataset:Function]\n\n # [DEF:rank_candidates:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Apply confidence ordering and determine best candidate per field.\n # @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n def rank_candidates(self, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n ranked = sorted(\n candidates,\n key=lambda candidate: (\n self._match_priority(candidate.get(\"match_type\")),\n -float(candidate.get(\"confidence_score\", 0.0)),\n int(candidate.get(\"candidate_rank\", 999)),\n ),\n )\n for index, candidate in enumerate(ranked, start=1):\n candidate[\"candidate_rank\"] = index\n return ranked\n # [/DEF:rank_candidates:Function]\n\n # [DEF:detect_conflicts:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Mark competing candidate sets that require explicit user review.\n def detect_conflicts(self, candidates: List[Dict[str, Any]]) -> bool:\n return len(candidates) > 1\n # [/DEF:detect_conflicts:Function]\n\n # [DEF:apply_field_decision:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Accept, reject, or manually override a field-level semantic value.\n def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> Dict[str, Any]:\n merged = dict(field_state)\n merged.update(decision)\n return merged\n # [/DEF:apply_field_decision:Function]\n\n # [DEF:propagate_source_version_update:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values.\n # @RELATION: [DEPENDS_ON] ->[SemanticSource]\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @PRE: source is persisted and fields belong to the same session aggregate.\n # @POST: unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched.\n # @SIDE_EFFECT: mutates in-memory field state for the caller to persist.\n # @DATA_CONTRACT: Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]]\n def propagate_source_version_update(\n self,\n source: SemanticSource,\n fields: Iterable[Any],\n ) -> Dict[str, int]:\n with belief_scope(\"SemanticSourceResolver.propagate_source_version_update\"):\n source_id = str(source.source_id or \"\").strip()\n source_version = str(source.source_version or \"\").strip()\n if not source_id or not source_version:\n logger.explore(\n \"Semantic source version propagation rejected due to incomplete source metadata\",\n extra={\"source_id\": source_id, \"source_version\": source_version},\n )\n raise ValueError(\"Semantic source must provide source_id and source_version\")\n\n propagated = 0\n preserved_locked = 0\n untouched = 0\n for field in fields:\n if str(getattr(field, \"source_id\", \"\") or \"\").strip() != source_id:\n untouched += 1\n continue\n if bool(getattr(field, \"is_locked\", False)) or getattr(field, \"provenance\", None) == FieldProvenance.MANUAL_OVERRIDE:\n preserved_locked += 1\n continue\n\n field.source_version = source_version\n field.needs_review = True\n field.has_conflict = bool(getattr(field, \"has_conflict\", False))\n propagated += 1\n\n logger.reflect(\n \"Semantic source version propagation completed\",\n extra={\n \"source_id\": source_id,\n \"source_version\": source_version,\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n },\n )\n return {\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n }\n # [/DEF:propagate_source_version_update:Function]\n\n # [DEF:_normalize_dictionary_row:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize one dictionary row into a consistent lookup structure.\n def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> Dict[str, Any]:\n field_name = (\n row.get(\"field_name\")\n or row.get(\"column_name\")\n or row.get(\"name\")\n or row.get(\"field\")\n )\n normalized_name = str(field_name or \"\").strip()\n return {\n \"field_name\": normalized_name,\n \"field_key\": self._normalize_key(normalized_name),\n \"verbose_name\": row.get(\"verbose_name\") or row.get(\"label\"),\n \"description\": row.get(\"description\"),\n \"display_format\": row.get(\"display_format\") or row.get(\"format\"),\n }\n # [/DEF:_normalize_dictionary_row:Function]\n\n # [DEF:_find_fuzzy_matches:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable.\n def _find_fuzzy_matches(self, field_name: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n normalized_target = self._normalize_key(field_name)\n fuzzy_matches: List[Dict[str, Any]] = []\n for row in rows:\n candidate_key = str(row.get(\"field_key\") or \"\")\n if not candidate_key:\n continue\n score = SequenceMatcher(None, normalized_target, candidate_key).ratio()\n if score < 0.72:\n continue\n fuzzy_matches.append({\"row\": row, \"score\": round(score, 3)})\n fuzzy_matches.sort(key=lambda item: item[\"score\"], reverse=True)\n return fuzzy_matches[:3]\n # [/DEF:_find_fuzzy_matches:Function]\n\n # [DEF:_build_candidate_payload:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Project normalized dictionary rows into semantic candidate payloads.\n def _build_candidate_payload(\n self,\n rank: int,\n match_type: CandidateMatchType,\n confidence_score: float,\n row: Mapping[str, Any],\n ) -> Dict[str, Any]:\n return {\n \"candidate_rank\": rank,\n \"match_type\": match_type.value,\n \"confidence_score\": confidence_score,\n \"proposed_verbose_name\": row.get(\"verbose_name\"),\n \"proposed_description\": row.get(\"description\"),\n \"proposed_display_format\": row.get(\"display_format\"),\n \"status\": CandidateStatus.PROPOSED.value,\n }\n # [/DEF:_build_candidate_payload:Function]\n\n # [DEF:_match_priority:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention.\n def _match_priority(self, match_type: Optional[str]) -> int:\n priority = {\n CandidateMatchType.EXACT.value: 0,\n CandidateMatchType.REFERENCE.value: 1,\n CandidateMatchType.FUZZY.value: 2,\n CandidateMatchType.GENERATED.value: 3,\n }\n return priority.get(str(match_type or \"\"), 99)\n # [/DEF:_match_priority:Function]\n\n # [DEF:_normalize_key:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons.\n def _normalize_key(self, value: str) -> str:\n return \"\".join(ch for ch in str(value or \"\").strip().lower() if ch.isalnum() or ch == \"_\")\n # [/DEF:_normalize_key:Function]\n# [/DEF:SemanticSourceResolver:Class]\n\n# [/DEF:SemanticSourceResolver:Module]\n" + "body": "# [DEF:SemanticSourceResolver:Module]\n# @COMPLEXITY: 4\n# @SEMANTICS: dataset_review, semantic_resolution, dictionary, trusted_sources, ranking\n# @PURPOSE: Resolve and rank semantic candidates from trusted dictionary-like sources before any inferred fallback.\n# @LAYER: Domain\n# @RELATION: [DEPENDS_ON] ->[LLMProviderService]\n# @RELATION: [DEPENDS_ON] ->[SemanticSource]\n# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n# @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n# @PRE: selected source and target field set must be known.\n# @POST: candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable.\n# @SIDE_EFFECT: may create conflict findings and semantic candidate records.\n# @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values.\n\nfrom __future__ import annotations\n\n# [DEF:imports:Block]\nfrom dataclasses import dataclass, field\nfrom difflib import SequenceMatcher\nfrom typing import Any, Dict, Iterable, List, Mapping, Optional\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.dataset_review import (\n CandidateMatchType,\n CandidateStatus,\n FieldProvenance,\n SemanticSource,\n)\n# [/DEF:imports:Block]\n\nlog = MarkerLogger(\"SemanticSourceResolver\")\n\n\n# [DEF:DictionaryResolutionResult:Class]\n# @COMPLEXITY: 2\n# @PURPOSE: Carries field-level dictionary resolution output with explicit review and partial-recovery state.\n@dataclass\nclass DictionaryResolutionResult:\n source_ref: str\n resolved_fields: List[Dict[str, Any]] = field(default_factory=list)\n unresolved_fields: List[str] = field(default_factory=list)\n partial_recovery: bool = False\n# [/DEF:DictionaryResolutionResult:Class]\n\n\n# [DEF:SemanticSourceResolver:Class]\n# @COMPLEXITY: 4\n# @PURPOSE: Resolve semantic candidates from trusted sources while preserving manual locks and confidence ordering.\n# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n# @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n# @PRE: source payload and target field collection are provided by the caller.\n# @POST: result contains confidence-ranked candidates and does not overwrite manual locks implicitly.\n# @SIDE_EFFECT: emits semantic trace logs for ranking and fallback decisions.\nclass SemanticSourceResolver:\n # [DEF:resolve_from_file:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize uploaded semantic file records into field-level candidates.\n def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult:\n return DictionaryResolutionResult(source_ref=str(source_payload.get(\"source_ref\") or \"uploaded_file\"))\n # [/DEF:resolve_from_file:Function]\n\n # [DEF:resolve_from_dictionary:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Resolve candidates from connected tabular dictionary sources.\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n # @PRE: dictionary source exists and fields contain stable field_name values.\n # @POST: returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit.\n # @SIDE_EFFECT: emits belief-state logs describing trusted-match and partial-recovery outcomes.\n # @DATA_CONTRACT: Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult]\n def resolve_from_dictionary(\n self,\n source_payload: Mapping[str, Any],\n fields: Iterable[Mapping[str, Any]],\n ) -> DictionaryResolutionResult:\n with belief_scope(\"SemanticSourceResolver.resolve_from_dictionary\"):\n source_ref = str(source_payload.get(\"source_ref\") or \"\").strip()\n dictionary_rows = source_payload.get(\"rows\")\n\n if not source_ref:\n log.explore(\"Dictionary semantic source is missing source_ref\", error=\"No source_ref in dictionary source\")\n raise ValueError(\"Dictionary semantic source must include source_ref\")\n\n if not isinstance(dictionary_rows, list) or not dictionary_rows:\n log.explore(\"Dictionary semantic source has no usable rows\", error=\"Dictionary rows is empty or not a list\", payload={\"source_ref\": source_ref})\n raise ValueError(\"Dictionary semantic source must include non-empty rows\")\n\n log.reason(\n \"Resolving semantics from trusted dictionary source\",\n payload={\"source_ref\": source_ref, \"row_count\": len(dictionary_rows)},\n )\n\n normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)]\n row_index = {\n row[\"field_key\"]: row\n for row in normalized_rows\n if row.get(\"field_key\")\n }\n\n resolved_fields: List[Dict[str, Any]] = []\n unresolved_fields: List[str] = []\n\n for raw_field in fields:\n field_name = str(raw_field.get(\"field_name\") or \"\").strip()\n if not field_name:\n continue\n\n is_locked = bool(raw_field.get(\"is_locked\"))\n if is_locked:\n log.reason(\n \"Preserving manual lock during dictionary resolution\",\n payload={\"field_name\": field_name},\n )\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.MANUAL_OVERRIDE.value,\n \"needs_review\": False,\n \"has_conflict\": False,\n \"is_locked\": True,\n \"status\": \"preserved_manual\",\n }\n )\n continue\n\n exact_match = row_index.get(self._normalize_key(field_name))\n candidates: List[Dict[str, Any]] = []\n\n if exact_match is not None:\n log.reason(\n \"Resolved exact dictionary match\",\n payload={\"field_name\": field_name, \"source_ref\": source_ref},\n )\n candidates.append(\n self._build_candidate_payload(\n rank=1,\n match_type=CandidateMatchType.EXACT,\n confidence_score=1.0,\n row=exact_match,\n )\n )\n else:\n fuzzy_matches = self._find_fuzzy_matches(field_name, normalized_rows)\n for rank_offset, fuzzy_match in enumerate(fuzzy_matches, start=1):\n candidates.append(\n self._build_candidate_payload(\n rank=rank_offset,\n match_type=CandidateMatchType.FUZZY,\n confidence_score=float(fuzzy_match[\"score\"]),\n row=fuzzy_match[\"row\"],\n )\n )\n\n if not candidates:\n unresolved_fields.append(field_name)\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.UNRESOLVED.value,\n \"needs_review\": True,\n \"has_conflict\": False,\n \"is_locked\": False,\n \"status\": \"unresolved\",\n }\n )\n log.explore(\"No trusted dictionary match found for field\", error=\"No dictionary match found for field\", payload={\"field_name\": field_name, \"source_ref\": source_ref})\n continue\n\n ranked_candidates = self.rank_candidates(candidates)\n applied_candidate = ranked_candidates[0]\n has_conflict = len(ranked_candidates) > 1\n provenance = (\n FieldProvenance.DICTIONARY_EXACT.value\n if applied_candidate[\"match_type\"] == CandidateMatchType.EXACT.value\n else FieldProvenance.FUZZY_INFERRED.value\n )\n needs_review = applied_candidate[\"match_type\"] != CandidateMatchType.EXACT.value\n\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": applied_candidate,\n \"candidates\": ranked_candidates,\n \"provenance\": provenance,\n \"needs_review\": needs_review,\n \"has_conflict\": has_conflict,\n \"is_locked\": False,\n \"status\": \"resolved\",\n }\n )\n\n result = DictionaryResolutionResult(\n source_ref=source_ref,\n resolved_fields=resolved_fields,\n unresolved_fields=unresolved_fields,\n partial_recovery=bool(unresolved_fields),\n )\n log.reflect(\n \"Dictionary resolution completed\",\n payload={\n \"source_ref\": source_ref,\n \"resolved_fields\": len(resolved_fields),\n \"unresolved_fields\": len(unresolved_fields),\n \"partial_recovery\": result.partial_recovery,\n },\n )\n return result\n # [/DEF:resolve_from_dictionary:Function]\n\n # [DEF:resolve_from_reference_dataset:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Reuse semantic metadata from trusted Superset datasets.\n def resolve_from_reference_dataset(\n self,\n source_payload: Mapping[str, Any],\n fields: Iterable[Mapping[str, Any]],\n ) -> DictionaryResolutionResult:\n return DictionaryResolutionResult(source_ref=str(source_payload.get(\"source_ref\") or \"reference_dataset\"))\n # [/DEF:resolve_from_reference_dataset:Function]\n\n # [DEF:rank_candidates:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Apply confidence ordering and determine best candidate per field.\n # @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n def rank_candidates(self, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n ranked = sorted(\n candidates,\n key=lambda candidate: (\n self._match_priority(candidate.get(\"match_type\")),\n -float(candidate.get(\"confidence_score\", 0.0)),\n int(candidate.get(\"candidate_rank\", 999)),\n ),\n )\n for index, candidate in enumerate(ranked, start=1):\n candidate[\"candidate_rank\"] = index\n return ranked\n # [/DEF:rank_candidates:Function]\n\n # [DEF:detect_conflicts:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Mark competing candidate sets that require explicit user review.\n def detect_conflicts(self, candidates: List[Dict[str, Any]]) -> bool:\n return len(candidates) > 1\n # [/DEF:detect_conflicts:Function]\n\n # [DEF:apply_field_decision:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Accept, reject, or manually override a field-level semantic value.\n def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> Dict[str, Any]:\n merged = dict(field_state)\n merged.update(decision)\n return merged\n # [/DEF:apply_field_decision:Function]\n\n # [DEF:propagate_source_version_update:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values.\n # @RELATION: [DEPENDS_ON] ->[SemanticSource]\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @PRE: source is persisted and fields belong to the same session aggregate.\n # @POST: unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched.\n # @SIDE_EFFECT: mutates in-memory field state for the caller to persist.\n # @DATA_CONTRACT: Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]]\n def propagate_source_version_update(\n self,\n source: SemanticSource,\n fields: Iterable[Any],\n ) -> Dict[str, int]:\n with belief_scope(\"SemanticSourceResolver.propagate_source_version_update\"):\n source_id = str(source.source_id or \"\").strip()\n source_version = str(source.source_version or \"\").strip()\n if not source_id or not source_version:\n log.explore(\"Semantic source version propagation rejected due to incomplete source metadata\", error=\"Source metadata incomplete (missing source_id or source_version)\", payload={\"source_id\": source_id, \"source_version\": source_version})\n raise ValueError(\"Semantic source must provide source_id and source_version\")\n\n propagated = 0\n preserved_locked = 0\n untouched = 0\n for field in fields:\n if str(getattr(field, \"source_id\", \"\") or \"\").strip() != source_id:\n untouched += 1\n continue\n if bool(getattr(field, \"is_locked\", False)) or getattr(field, \"provenance\", None) == FieldProvenance.MANUAL_OVERRIDE:\n preserved_locked += 1\n continue\n\n field.source_version = source_version\n field.needs_review = True\n field.has_conflict = bool(getattr(field, \"has_conflict\", False))\n propagated += 1\n\n log.reflect(\n \"Semantic source version propagation completed\",\n payload={\n \"source_id\": source_id,\n \"source_version\": source_version,\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n },\n )\n return {\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n }\n # [/DEF:propagate_source_version_update:Function]\n\n # [DEF:_normalize_dictionary_row:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize one dictionary row into a consistent lookup structure.\n def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> Dict[str, Any]:\n field_name = (\n row.get(\"field_name\")\n or row.get(\"column_name\")\n or row.get(\"name\")\n or row.get(\"field\")\n )\n normalized_name = str(field_name or \"\").strip()\n return {\n \"field_name\": normalized_name,\n \"field_key\": self._normalize_key(normalized_name),\n \"verbose_name\": row.get(\"verbose_name\") or row.get(\"label\"),\n \"description\": row.get(\"description\"),\n \"display_format\": row.get(\"display_format\") or row.get(\"format\"),\n }\n # [/DEF:_normalize_dictionary_row:Function]\n\n # [DEF:_find_fuzzy_matches:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable.\n def _find_fuzzy_matches(self, field_name: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n normalized_target = self._normalize_key(field_name)\n fuzzy_matches: List[Dict[str, Any]] = []\n for row in rows:\n candidate_key = str(row.get(\"field_key\") or \"\")\n if not candidate_key:\n continue\n score = SequenceMatcher(None, normalized_target, candidate_key).ratio()\n if score < 0.72:\n continue\n fuzzy_matches.append({\"row\": row, \"score\": round(score, 3)})\n fuzzy_matches.sort(key=lambda item: item[\"score\"], reverse=True)\n return fuzzy_matches[:3]\n # [/DEF:_find_fuzzy_matches:Function]\n\n # [DEF:_build_candidate_payload:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Project normalized dictionary rows into semantic candidate payloads.\n def _build_candidate_payload(\n self,\n rank: int,\n match_type: CandidateMatchType,\n confidence_score: float,\n row: Mapping[str, Any],\n ) -> Dict[str, Any]:\n return {\n \"candidate_rank\": rank,\n \"match_type\": match_type.value,\n \"confidence_score\": confidence_score,\n \"proposed_verbose_name\": row.get(\"verbose_name\"),\n \"proposed_description\": row.get(\"description\"),\n \"proposed_display_format\": row.get(\"display_format\"),\n \"status\": CandidateStatus.PROPOSED.value,\n }\n # [/DEF:_build_candidate_payload:Function]\n\n # [DEF:_match_priority:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention.\n def _match_priority(self, match_type: Optional[str]) -> int:\n priority = {\n CandidateMatchType.EXACT.value: 0,\n CandidateMatchType.REFERENCE.value: 1,\n CandidateMatchType.FUZZY.value: 2,\n CandidateMatchType.GENERATED.value: 3,\n }\n return priority.get(str(match_type or \"\"), 99)\n # [/DEF:_match_priority:Function]\n\n # [DEF:_normalize_key:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons.\n def _normalize_key(self, value: str) -> str:\n return \"\".join(ch for ch in str(value or \"\").strip().lower() if ch.isalnum() or ch == \"_\")\n # [/DEF:_normalize_key:Function]\n# [/DEF:SemanticSourceResolver:Class]\n\n# [/DEF:SemanticSourceResolver:Module]\n" }, { "contract_id": "imports", "contract_type": "Block", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", "start_line": 17, - "end_line": 29, + "end_line": 30, "tier": "TIER_1", "complexity": 1, "metadata": {}, @@ -83229,14 +84238,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:imports:Block]\nfrom dataclasses import dataclass, field\nfrom difflib import SequenceMatcher\nfrom typing import Any, Dict, Iterable, List, Mapping, Optional\n\nfrom src.core.logger import belief_scope, logger\nfrom src.models.dataset_review import (\n CandidateMatchType,\n CandidateStatus,\n FieldProvenance,\n SemanticSource,\n)\n# [/DEF:imports:Block]\n" + "body": "# [DEF:imports:Block]\nfrom dataclasses import dataclass, field\nfrom difflib import SequenceMatcher\nfrom typing import Any, Dict, Iterable, List, Mapping, Optional\n\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\nfrom src.models.dataset_review import (\n CandidateMatchType,\n CandidateStatus,\n FieldProvenance,\n SemanticSource,\n)\n# [/DEF:imports:Block]\n" }, { "contract_id": "DictionaryResolutionResult", "contract_type": "Class", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 32, - "end_line": 41, + "start_line": 35, + "end_line": 44, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83252,8 +84261,8 @@ "contract_id": "resolve_from_file", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 53, - "end_line": 58, + "start_line": 56, + "end_line": 61, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83269,8 +84278,8 @@ "contract_id": "resolve_from_dictionary", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 60, - "end_line": 216, + "start_line": 63, + "end_line": 213, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -83341,14 +84350,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:resolve_from_dictionary:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Resolve candidates from connected tabular dictionary sources.\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n # @PRE: dictionary source exists and fields contain stable field_name values.\n # @POST: returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit.\n # @SIDE_EFFECT: emits belief-state logs describing trusted-match and partial-recovery outcomes.\n # @DATA_CONTRACT: Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult]\n def resolve_from_dictionary(\n self,\n source_payload: Mapping[str, Any],\n fields: Iterable[Mapping[str, Any]],\n ) -> DictionaryResolutionResult:\n with belief_scope(\"SemanticSourceResolver.resolve_from_dictionary\"):\n source_ref = str(source_payload.get(\"source_ref\") or \"\").strip()\n dictionary_rows = source_payload.get(\"rows\")\n\n if not source_ref:\n logger.explore(\"Dictionary semantic source is missing source_ref\")\n raise ValueError(\"Dictionary semantic source must include source_ref\")\n\n if not isinstance(dictionary_rows, list) or not dictionary_rows:\n logger.explore(\n \"Dictionary semantic source has no usable rows\",\n extra={\"source_ref\": source_ref},\n )\n raise ValueError(\"Dictionary semantic source must include non-empty rows\")\n\n logger.reason(\n \"Resolving semantics from trusted dictionary source\",\n extra={\"source_ref\": source_ref, \"row_count\": len(dictionary_rows)},\n )\n\n normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)]\n row_index = {\n row[\"field_key\"]: row\n for row in normalized_rows\n if row.get(\"field_key\")\n }\n\n resolved_fields: List[Dict[str, Any]] = []\n unresolved_fields: List[str] = []\n\n for raw_field in fields:\n field_name = str(raw_field.get(\"field_name\") or \"\").strip()\n if not field_name:\n continue\n\n is_locked = bool(raw_field.get(\"is_locked\"))\n if is_locked:\n logger.reason(\n \"Preserving manual lock during dictionary resolution\",\n extra={\"field_name\": field_name},\n )\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.MANUAL_OVERRIDE.value,\n \"needs_review\": False,\n \"has_conflict\": False,\n \"is_locked\": True,\n \"status\": \"preserved_manual\",\n }\n )\n continue\n\n exact_match = row_index.get(self._normalize_key(field_name))\n candidates: List[Dict[str, Any]] = []\n\n if exact_match is not None:\n logger.reason(\n \"Resolved exact dictionary match\",\n extra={\"field_name\": field_name, \"source_ref\": source_ref},\n )\n candidates.append(\n self._build_candidate_payload(\n rank=1,\n match_type=CandidateMatchType.EXACT,\n confidence_score=1.0,\n row=exact_match,\n )\n )\n else:\n fuzzy_matches = self._find_fuzzy_matches(field_name, normalized_rows)\n for rank_offset, fuzzy_match in enumerate(fuzzy_matches, start=1):\n candidates.append(\n self._build_candidate_payload(\n rank=rank_offset,\n match_type=CandidateMatchType.FUZZY,\n confidence_score=float(fuzzy_match[\"score\"]),\n row=fuzzy_match[\"row\"],\n )\n )\n\n if not candidates:\n unresolved_fields.append(field_name)\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.UNRESOLVED.value,\n \"needs_review\": True,\n \"has_conflict\": False,\n \"is_locked\": False,\n \"status\": \"unresolved\",\n }\n )\n logger.explore(\n \"No trusted dictionary match found for field\",\n extra={\"field_name\": field_name, \"source_ref\": source_ref},\n )\n continue\n\n ranked_candidates = self.rank_candidates(candidates)\n applied_candidate = ranked_candidates[0]\n has_conflict = len(ranked_candidates) > 1\n provenance = (\n FieldProvenance.DICTIONARY_EXACT.value\n if applied_candidate[\"match_type\"] == CandidateMatchType.EXACT.value\n else FieldProvenance.FUZZY_INFERRED.value\n )\n needs_review = applied_candidate[\"match_type\"] != CandidateMatchType.EXACT.value\n\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": applied_candidate,\n \"candidates\": ranked_candidates,\n \"provenance\": provenance,\n \"needs_review\": needs_review,\n \"has_conflict\": has_conflict,\n \"is_locked\": False,\n \"status\": \"resolved\",\n }\n )\n\n result = DictionaryResolutionResult(\n source_ref=source_ref,\n resolved_fields=resolved_fields,\n unresolved_fields=unresolved_fields,\n partial_recovery=bool(unresolved_fields),\n )\n logger.reflect(\n \"Dictionary resolution completed\",\n extra={\n \"source_ref\": source_ref,\n \"resolved_fields\": len(resolved_fields),\n \"unresolved_fields\": len(unresolved_fields),\n \"partial_recovery\": result.partial_recovery,\n },\n )\n return result\n # [/DEF:resolve_from_dictionary:Function]\n" + "body": " # [DEF:resolve_from_dictionary:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Resolve candidates from connected tabular dictionary sources.\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @RELATION: [DEPENDS_ON] ->[SemanticCandidate]\n # @PRE: dictionary source exists and fields contain stable field_name values.\n # @POST: returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit.\n # @SIDE_EFFECT: emits belief-state logs describing trusted-match and partial-recovery outcomes.\n # @DATA_CONTRACT: Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult]\n def resolve_from_dictionary(\n self,\n source_payload: Mapping[str, Any],\n fields: Iterable[Mapping[str, Any]],\n ) -> DictionaryResolutionResult:\n with belief_scope(\"SemanticSourceResolver.resolve_from_dictionary\"):\n source_ref = str(source_payload.get(\"source_ref\") or \"\").strip()\n dictionary_rows = source_payload.get(\"rows\")\n\n if not source_ref:\n log.explore(\"Dictionary semantic source is missing source_ref\", error=\"No source_ref in dictionary source\")\n raise ValueError(\"Dictionary semantic source must include source_ref\")\n\n if not isinstance(dictionary_rows, list) or not dictionary_rows:\n log.explore(\"Dictionary semantic source has no usable rows\", error=\"Dictionary rows is empty or not a list\", payload={\"source_ref\": source_ref})\n raise ValueError(\"Dictionary semantic source must include non-empty rows\")\n\n log.reason(\n \"Resolving semantics from trusted dictionary source\",\n payload={\"source_ref\": source_ref, \"row_count\": len(dictionary_rows)},\n )\n\n normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)]\n row_index = {\n row[\"field_key\"]: row\n for row in normalized_rows\n if row.get(\"field_key\")\n }\n\n resolved_fields: List[Dict[str, Any]] = []\n unresolved_fields: List[str] = []\n\n for raw_field in fields:\n field_name = str(raw_field.get(\"field_name\") or \"\").strip()\n if not field_name:\n continue\n\n is_locked = bool(raw_field.get(\"is_locked\"))\n if is_locked:\n log.reason(\n \"Preserving manual lock during dictionary resolution\",\n payload={\"field_name\": field_name},\n )\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.MANUAL_OVERRIDE.value,\n \"needs_review\": False,\n \"has_conflict\": False,\n \"is_locked\": True,\n \"status\": \"preserved_manual\",\n }\n )\n continue\n\n exact_match = row_index.get(self._normalize_key(field_name))\n candidates: List[Dict[str, Any]] = []\n\n if exact_match is not None:\n log.reason(\n \"Resolved exact dictionary match\",\n payload={\"field_name\": field_name, \"source_ref\": source_ref},\n )\n candidates.append(\n self._build_candidate_payload(\n rank=1,\n match_type=CandidateMatchType.EXACT,\n confidence_score=1.0,\n row=exact_match,\n )\n )\n else:\n fuzzy_matches = self._find_fuzzy_matches(field_name, normalized_rows)\n for rank_offset, fuzzy_match in enumerate(fuzzy_matches, start=1):\n candidates.append(\n self._build_candidate_payload(\n rank=rank_offset,\n match_type=CandidateMatchType.FUZZY,\n confidence_score=float(fuzzy_match[\"score\"]),\n row=fuzzy_match[\"row\"],\n )\n )\n\n if not candidates:\n unresolved_fields.append(field_name)\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": None,\n \"candidates\": [],\n \"provenance\": FieldProvenance.UNRESOLVED.value,\n \"needs_review\": True,\n \"has_conflict\": False,\n \"is_locked\": False,\n \"status\": \"unresolved\",\n }\n )\n log.explore(\"No trusted dictionary match found for field\", error=\"No dictionary match found for field\", payload={\"field_name\": field_name, \"source_ref\": source_ref})\n continue\n\n ranked_candidates = self.rank_candidates(candidates)\n applied_candidate = ranked_candidates[0]\n has_conflict = len(ranked_candidates) > 1\n provenance = (\n FieldProvenance.DICTIONARY_EXACT.value\n if applied_candidate[\"match_type\"] == CandidateMatchType.EXACT.value\n else FieldProvenance.FUZZY_INFERRED.value\n )\n needs_review = applied_candidate[\"match_type\"] != CandidateMatchType.EXACT.value\n\n resolved_fields.append(\n {\n \"field_name\": field_name,\n \"applied_candidate\": applied_candidate,\n \"candidates\": ranked_candidates,\n \"provenance\": provenance,\n \"needs_review\": needs_review,\n \"has_conflict\": has_conflict,\n \"is_locked\": False,\n \"status\": \"resolved\",\n }\n )\n\n result = DictionaryResolutionResult(\n source_ref=source_ref,\n resolved_fields=resolved_fields,\n unresolved_fields=unresolved_fields,\n partial_recovery=bool(unresolved_fields),\n )\n log.reflect(\n \"Dictionary resolution completed\",\n payload={\n \"source_ref\": source_ref,\n \"resolved_fields\": len(resolved_fields),\n \"unresolved_fields\": len(unresolved_fields),\n \"partial_recovery\": result.partial_recovery,\n },\n )\n return result\n # [/DEF:resolve_from_dictionary:Function]\n" }, { "contract_id": "resolve_from_reference_dataset", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 218, - "end_line": 227, + "start_line": 215, + "end_line": 224, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83364,8 +84373,8 @@ "contract_id": "rank_candidates", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 229, - "end_line": 245, + "start_line": 226, + "end_line": 242, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83415,8 +84424,8 @@ "contract_id": "detect_conflicts", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 247, - "end_line": 252, + "start_line": 244, + "end_line": 249, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83432,8 +84441,8 @@ "contract_id": "apply_field_decision", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 254, - "end_line": 261, + "start_line": 251, + "end_line": 258, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83449,8 +84458,8 @@ "contract_id": "propagate_source_version_update", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 263, - "end_line": 318, + "start_line": 260, + "end_line": 312, "tier": "TIER_2", "complexity": 4, "metadata": { @@ -83521,14 +84530,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:propagate_source_version_update:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values.\n # @RELATION: [DEPENDS_ON] ->[SemanticSource]\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @PRE: source is persisted and fields belong to the same session aggregate.\n # @POST: unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched.\n # @SIDE_EFFECT: mutates in-memory field state for the caller to persist.\n # @DATA_CONTRACT: Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]]\n def propagate_source_version_update(\n self,\n source: SemanticSource,\n fields: Iterable[Any],\n ) -> Dict[str, int]:\n with belief_scope(\"SemanticSourceResolver.propagate_source_version_update\"):\n source_id = str(source.source_id or \"\").strip()\n source_version = str(source.source_version or \"\").strip()\n if not source_id or not source_version:\n logger.explore(\n \"Semantic source version propagation rejected due to incomplete source metadata\",\n extra={\"source_id\": source_id, \"source_version\": source_version},\n )\n raise ValueError(\"Semantic source must provide source_id and source_version\")\n\n propagated = 0\n preserved_locked = 0\n untouched = 0\n for field in fields:\n if str(getattr(field, \"source_id\", \"\") or \"\").strip() != source_id:\n untouched += 1\n continue\n if bool(getattr(field, \"is_locked\", False)) or getattr(field, \"provenance\", None) == FieldProvenance.MANUAL_OVERRIDE:\n preserved_locked += 1\n continue\n\n field.source_version = source_version\n field.needs_review = True\n field.has_conflict = bool(getattr(field, \"has_conflict\", False))\n propagated += 1\n\n logger.reflect(\n \"Semantic source version propagation completed\",\n extra={\n \"source_id\": source_id,\n \"source_version\": source_version,\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n },\n )\n return {\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n }\n # [/DEF:propagate_source_version_update:Function]\n" + "body": " # [DEF:propagate_source_version_update:Function]\n # @COMPLEXITY: 4\n # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values.\n # @RELATION: [DEPENDS_ON] ->[SemanticSource]\n # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]\n # @PRE: source is persisted and fields belong to the same session aggregate.\n # @POST: unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched.\n # @SIDE_EFFECT: mutates in-memory field state for the caller to persist.\n # @DATA_CONTRACT: Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]]\n def propagate_source_version_update(\n self,\n source: SemanticSource,\n fields: Iterable[Any],\n ) -> Dict[str, int]:\n with belief_scope(\"SemanticSourceResolver.propagate_source_version_update\"):\n source_id = str(source.source_id or \"\").strip()\n source_version = str(source.source_version or \"\").strip()\n if not source_id or not source_version:\n log.explore(\"Semantic source version propagation rejected due to incomplete source metadata\", error=\"Source metadata incomplete (missing source_id or source_version)\", payload={\"source_id\": source_id, \"source_version\": source_version})\n raise ValueError(\"Semantic source must provide source_id and source_version\")\n\n propagated = 0\n preserved_locked = 0\n untouched = 0\n for field in fields:\n if str(getattr(field, \"source_id\", \"\") or \"\").strip() != source_id:\n untouched += 1\n continue\n if bool(getattr(field, \"is_locked\", False)) or getattr(field, \"provenance\", None) == FieldProvenance.MANUAL_OVERRIDE:\n preserved_locked += 1\n continue\n\n field.source_version = source_version\n field.needs_review = True\n field.has_conflict = bool(getattr(field, \"has_conflict\", False))\n propagated += 1\n\n log.reflect(\n \"Semantic source version propagation completed\",\n payload={\n \"source_id\": source_id,\n \"source_version\": source_version,\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n },\n )\n return {\n \"propagated\": propagated,\n \"preserved_locked\": preserved_locked,\n \"untouched\": untouched,\n }\n # [/DEF:propagate_source_version_update:Function]\n" }, { "contract_id": "_normalize_dictionary_row", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 320, - "end_line": 338, + "start_line": 314, + "end_line": 332, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83544,8 +84553,8 @@ "contract_id": "_find_fuzzy_matches", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 340, - "end_line": 356, + "start_line": 334, + "end_line": 350, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83561,8 +84570,8 @@ "contract_id": "_build_candidate_payload", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 358, - "end_line": 377, + "start_line": 352, + "end_line": 371, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83578,8 +84587,8 @@ "contract_id": "_match_priority", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 379, - "end_line": 390, + "start_line": 373, + "end_line": 384, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83595,8 +84604,8 @@ "contract_id": "_normalize_key", "contract_type": "Function", "file_path": "backend/src/services/dataset_review/semantic_resolver.py", - "start_line": 392, - "end_line": 397, + "start_line": 386, + "end_line": 391, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -83721,7 +84730,7 @@ "contract_type": "Module", "file_path": "backend/src/services/git/_base.py", "start_line": 1, - "end_line": 319, + "end_line": 318, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -83782,14 +84791,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitServiceBase:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, service, repository, version_control, initialization\n# @PURPOSE: Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), and identity configuration.\n# @RELATION: INHERITED_BY -> [GitService]\n# @RELATION: DEPENDS_ON -> [SessionLocal]\n# @RELATION: DEPENDS_ON -> [AppConfigRecord]\n# @RELATION: DEPENDS_ON -> [GitRepository]\n\nimport os\nimport re\nimport shutil\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional\nfrom git import Repo\nfrom git.exc import InvalidGitRepositoryError, NoSuchPathError\nfrom fastapi import HTTPException\nfrom src.core.logger import logger, belief_scope\nfrom src.models.git import GitRepository\nfrom src.models.config import AppConfigRecord\nfrom src.core.database import SessionLocal\n\n\n# [DEF:GitServiceBase:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Base class for GitService providing initialization, path resolution, repository lifecycle, and identity.\nclass GitServiceBase:\n # [DEF:GitService_init:Function]\n # @PURPOSE: Initializes the GitService with a base path for repositories.\n # @PARAM: base_path (str) - Root directory for all Git clones.\n # @PRE: base_path is a valid string path.\n # @POST: GitService is initialized; base_path directory exists.\n def __init__(self, base_path: str = \"git_repos\"):\n with belief_scope(\"GitService.__init__\"):\n backend_root = Path(__file__).parents[3]\n self.legacy_base_path = str((backend_root / \"git_repos\").resolve())\n self._uses_default_base_path = base_path == \"git_repos\"\n self.base_path = self._resolve_base_path(base_path)\n self._ensure_base_path_exists()\n # [/DEF:GitService_init:Function]\n\n # [DEF:_ensure_base_path_exists:Function]\n # @PURPOSE: Ensure the repositories root directory exists and is a directory.\n # @PRE: self.base_path is resolved to filesystem path.\n # @POST: self.base_path exists as directory or raises ValueError.\n def _ensure_base_path_exists(self) -> None:\n base = Path(self.base_path)\n if base.exists() and not base.is_dir():\n raise ValueError(f\"Git repositories base path is not a directory: {self.base_path}\")\n try:\n base.mkdir(parents=True, exist_ok=True)\n except (PermissionError, OSError) as e:\n logger.warning(\n f\"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}\"\n )\n raise ValueError(f\"Cannot create Git repositories base path: {self.base_path}. {e}\")\n # [/DEF:_ensure_base_path_exists:Function]\n\n # [DEF:_resolve_base_path:Function]\n # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings.\n # @PRE: base_path is a string path.\n # @POST: Returns absolute path for Git repositories root.\n # @RETURN: str\n def _resolve_base_path(self, base_path: str) -> str:\n backend_root = Path(__file__).parents[3]\n fallback_path = str((backend_root / base_path).resolve())\n if base_path != \"git_repos\":\n return fallback_path\n try:\n session = SessionLocal()\n try:\n config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n finally:\n session.close()\n payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {}\n storage_cfg = payload.get(\"settings\", {}).get(\"storage\", {}) if isinstance(payload, dict) else {}\n root_path = str(storage_cfg.get(\"root_path\", \"\")).strip()\n repo_path = str(storage_cfg.get(\"repo_path\", \"\")).strip()\n if not root_path:\n return fallback_path\n project_root = Path(__file__).parents[4]\n root = Path(root_path)\n if not root.is_absolute():\n root = (project_root / root).resolve()\n repo_root = Path(repo_path) if repo_path else Path(\"repositorys\")\n if repo_root.is_absolute():\n return str(repo_root.resolve())\n return str((root / repo_root).resolve())\n except Exception as e:\n logger.warning(f\"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}\")\n return fallback_path\n # [/DEF:_resolve_base_path:Function]\n\n # [DEF:_normalize_repo_key:Function]\n # @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name.\n # @PRE: repo_key can be None/empty.\n # @POST: Returns normalized non-empty key.\n # @RETURN: str\n def _normalize_repo_key(self, repo_key: Optional[str]) -> str:\n raw_key = str(repo_key or \"\").strip().lower()\n normalized = re.sub(r\"[^a-z0-9._-]+\", \"-\", raw_key).strip(\"._-\")\n return normalized or \"dashboard\"\n # [/DEF:_normalize_repo_key:Function]\n\n # [DEF:_update_repo_local_path:Function]\n # @PURPOSE: Persist repository local_path in GitRepository table when record exists.\n # @PRE: dashboard_id is valid integer.\n # @POST: local_path is updated for existing record.\n def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None:\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.local_path = local_path\n session.commit()\n finally:\n session.close()\n except Exception as e:\n logger.warning(f\"[_update_repo_local_path][Coherence:Failed] {e}\")\n # [/DEF:_update_repo_local_path:Function]\n\n # [DEF:_migrate_repo_directory:Function]\n # @PURPOSE: Move legacy repository directory to target path and sync DB metadata.\n # @PRE: source_path exists.\n # @POST: Repository content available at target_path.\n # @RETURN: str\n def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str:\n source_abs = os.path.abspath(source_path)\n target_abs = os.path.abspath(target_path)\n if source_abs == target_abs:\n return source_abs\n if os.path.exists(target_abs):\n logger.warning(f\"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}\")\n return source_abs\n Path(target_abs).parent.mkdir(parents=True, exist_ok=True)\n try:\n os.replace(source_abs, target_abs)\n except OSError:\n shutil.move(source_abs, target_abs)\n self._update_repo_local_path(dashboard_id, target_abs)\n logger.info(\n f\"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}\"\n )\n return target_abs\n # [/DEF:_migrate_repo_directory:Function]\n\n # [DEF:_get_repo_path:Function]\n # @PURPOSE: Resolves the local filesystem path for a dashboard's repository.\n # @PARAM: dashboard_id (int)\n # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent.\n # @PRE: dashboard_id is an integer.\n # @POST: Returns DB-local_path when present, otherwise base_path/.\n # @RETURN: str\n def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str:\n with belief_scope(\"GitService._get_repo_path\"):\n if dashboard_id is None:\n raise ValueError(\"dashboard_id cannot be None\")\n self._ensure_base_path_exists()\n fallback_key = repo_key if repo_key is not None else str(dashboard_id)\n normalized_key = self._normalize_repo_key(fallback_key)\n target_path = os.path.join(self.base_path, normalized_key)\n if not self._uses_default_base_path:\n return target_path\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n finally:\n session.close()\n if db_repo and db_repo.local_path:\n db_path = os.path.abspath(db_repo.local_path)\n if os.path.exists(db_path):\n if (\n os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path)\n and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep)\n ):\n return self._migrate_repo_directory(dashboard_id, db_path, target_path)\n return db_path\n except Exception as e:\n logger.warning(f\"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}\")\n legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id))\n if os.path.exists(legacy_id_path) and not os.path.exists(target_path):\n return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path)\n if os.path.exists(target_path):\n self._update_repo_local_path(dashboard_id, target_path)\n return target_path\n # [/DEF:_get_repo_path:Function]\n\n # [DEF:init_repo:Function]\n # @PURPOSE: Initialize or clone a repository for a dashboard.\n # @PARAM: dashboard_id (int), remote_url (str), pat (str), repo_key (Optional[str]).\n # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided.\n # @POST: Repository is cloned or opened at the local path.\n # @RETURN: Repo\n def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo:\n with belief_scope(\"GitService.init_repo\"):\n self._ensure_base_path_exists()\n repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id))\n Path(repo_path).parent.mkdir(parents=True, exist_ok=True)\n if pat and \"://\" in remote_url:\n proto, rest = remote_url.split(\"://\", 1)\n auth_url = f\"{proto}://oauth2:{pat}@{rest}\"\n else:\n auth_url = remote_url\n if os.path.exists(repo_path):\n logger.info(f\"[init_repo][Action] Opening existing repo at {repo_path}\")\n try:\n repo = Repo(repo_path)\n except (InvalidGitRepositoryError, NoSuchPathError):\n logger.warning(f\"[init_repo][Action] Existing path is not a Git repository, recreating: {repo_path}\")\n stale_path = Path(repo_path)\n if stale_path.exists():\n shutil.rmtree(stale_path, ignore_errors=True)\n if stale_path.exists():\n try:\n stale_path.unlink()\n except Exception:\n pass\n repo = Repo.clone_from(auth_url, repo_path)\n self._ensure_gitflow_branches(repo, dashboard_id)\n return repo\n logger.info(f\"[init_repo][Action] Cloning {remote_url} to {repo_path}\")\n repo = Repo.clone_from(auth_url, repo_path)\n self._ensure_gitflow_branches(repo, dashboard_id)\n return repo\n # [/DEF:init_repo:Function]\n\n # [DEF:delete_repo:Function]\n # @PURPOSE: Remove local repository and DB binding for a dashboard.\n # @PRE: dashboard_id is a valid integer.\n # @POST: Local path is deleted when present and GitRepository row is removed.\n def delete_repo(self, dashboard_id: int) -> None:\n with belief_scope(\"GitService.delete_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n removed_files = False\n if os.path.exists(repo_path):\n if os.path.isdir(repo_path):\n shutil.rmtree(repo_path)\n else:\n os.remove(repo_path)\n removed_files = True\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n session.delete(db_repo)\n session.commit()\n return\n if removed_files:\n return\n raise HTTPException(\n status_code=404,\n detail=f\"Repository for dashboard {dashboard_id} not found\",\n )\n except HTTPException:\n session.rollback()\n raise\n except Exception as e:\n session.rollback()\n logger.error(f\"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}\")\n raise HTTPException(status_code=500, detail=f\"Failed to delete repository: {str(e)}\")\n finally:\n session.close()\n # [/DEF:delete_repo:Function]\n\n # [DEF:get_repo:Function]\n # @PURPOSE: Get Repo object for a dashboard.\n # @PRE: Repository must exist on disk for the given dashboard_id.\n # @POST: Returns a GitPython Repo instance for the dashboard.\n # @RETURN: Repo\n def get_repo(self, dashboard_id: int) -> Repo:\n with belief_scope(\"GitService.get_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n logger.error(f\"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist\")\n raise HTTPException(status_code=404, detail=f\"Repository for dashboard {dashboard_id} not found\")\n try:\n return Repo(repo_path)\n except Exception as e:\n logger.error(f\"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to open local Git repository\")\n # [/DEF:get_repo:Function]\n\n # [DEF:configure_identity:Function]\n # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations.\n # @PRE: dashboard_id repository exists; git_username/git_email may be empty.\n # @POST: Repository config has user.name and user.email when both identity values are provided.\n def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None:\n with belief_scope(\"GitService.configure_identity\"):\n normalized_username = str(git_username or \"\").strip()\n normalized_email = str(git_email or \"\").strip()\n if not normalized_username or not normalized_email:\n return\n repo = self.get_repo(dashboard_id)\n try:\n with repo.config_writer(config_level=\"repository\") as config_writer:\n config_writer.set_value(\"user\", \"name\", normalized_username)\n config_writer.set_value(\"user\", \"email\", normalized_email)\n logger.info(\"[configure_identity][Action] Applied repository-local git identity for dashboard %s\", dashboard_id)\n except Exception as e:\n logger.error(f\"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}\")\n raise HTTPException(status_code=500, detail=f\"Failed to configure git identity: {str(e)}\")\n # [/DEF:configure_identity:Function]\n# [/DEF:GitServiceBase:Class]\n# [/DEF:GitServiceBase:Module]\n" + "body": "# [DEF:GitServiceBase:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, service, repository, version_control, initialization\n# @PURPOSE: Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), and identity configuration.\n# @RELATION: INHERITED_BY -> [GitService]\n# @RELATION: DEPENDS_ON -> [SessionLocal]\n# @RELATION: DEPENDS_ON -> [AppConfigRecord]\n# @RELATION: DEPENDS_ON -> [GitRepository]\n\nimport os\nimport re\nimport shutil\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional\nfrom git import Repo\nfrom git.exc import InvalidGitRepositoryError, NoSuchPathError\nfrom fastapi import HTTPException\nfrom src.core.logger import belief_scope\nfrom src.core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"GitBase\")\nfrom src.models.git import GitRepository\nfrom src.models.config import AppConfigRecord\nfrom src.core.database import SessionLocal\n\n\n# [DEF:GitServiceBase:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Base class for GitService providing initialization, path resolution, repository lifecycle, and identity.\nclass GitServiceBase:\n # [DEF:GitService_init:Function]\n # @PURPOSE: Initializes the GitService with a base path for repositories.\n # @PARAM: base_path (str) - Root directory for all Git clones.\n # @PRE: base_path is a valid string path.\n # @POST: GitService is initialized; base_path directory exists.\n def __init__(self, base_path: str = \"git_repos\"):\n with belief_scope(\"GitService.__init__\"):\n backend_root = Path(__file__).parents[3]\n self.legacy_base_path = str((backend_root / \"git_repos\").resolve())\n self._uses_default_base_path = base_path == \"git_repos\"\n self.base_path = self._resolve_base_path(base_path)\n self._ensure_base_path_exists()\n # [/DEF:GitService_init:Function]\n\n # [DEF:_ensure_base_path_exists:Function]\n # @PURPOSE: Ensure the repositories root directory exists and is a directory.\n # @PRE: self.base_path is resolved to filesystem path.\n # @POST: self.base_path exists as directory or raises ValueError.\n def _ensure_base_path_exists(self) -> None:\n base = Path(self.base_path)\n if base.exists() and not base.is_dir():\n raise ValueError(f\"Git repositories base path is not a directory: {self.base_path}\")\n try:\n base.mkdir(parents=True, exist_ok=True)\n except (PermissionError, OSError) as e:\n log.explore(\"Cannot create Git repositories base path\", payload={\"base_path\": self.base_path}, error=str(e))\n raise ValueError(f\"Cannot create Git repositories base path: {self.base_path}. {e}\")\n # [/DEF:_ensure_base_path_exists:Function]\n\n # [DEF:_resolve_base_path:Function]\n # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings.\n # @PRE: base_path is a string path.\n # @POST: Returns absolute path for Git repositories root.\n # @RETURN: str\n def _resolve_base_path(self, base_path: str) -> str:\n backend_root = Path(__file__).parents[3]\n fallback_path = str((backend_root / base_path).resolve())\n if base_path != \"git_repos\":\n return fallback_path\n try:\n session = SessionLocal()\n try:\n config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n finally:\n session.close()\n payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {}\n storage_cfg = payload.get(\"settings\", {}).get(\"storage\", {}) if isinstance(payload, dict) else {}\n root_path = str(storage_cfg.get(\"root_path\", \"\")).strip()\n repo_path = str(storage_cfg.get(\"repo_path\", \"\")).strip()\n if not root_path:\n return fallback_path\n project_root = Path(__file__).parents[4]\n root = Path(root_path)\n if not root.is_absolute():\n root = (project_root / root).resolve()\n repo_root = Path(repo_path) if repo_path else Path(\"repositorys\")\n if repo_root.is_absolute():\n return str(repo_root.resolve())\n return str((root / repo_root).resolve())\n except Exception as e:\n log.explore(\"Falling back to default base path\", error=str(e))\n return fallback_path\n # [/DEF:_resolve_base_path:Function]\n\n # [DEF:_normalize_repo_key:Function]\n # @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name.\n # @PRE: repo_key can be None/empty.\n # @POST: Returns normalized non-empty key.\n # @RETURN: str\n def _normalize_repo_key(self, repo_key: Optional[str]) -> str:\n raw_key = str(repo_key or \"\").strip().lower()\n normalized = re.sub(r\"[^a-z0-9._-]+\", \"-\", raw_key).strip(\"._-\")\n return normalized or \"dashboard\"\n # [/DEF:_normalize_repo_key:Function]\n\n # [DEF:_update_repo_local_path:Function]\n # @PURPOSE: Persist repository local_path in GitRepository table when record exists.\n # @PRE: dashboard_id is valid integer.\n # @POST: local_path is updated for existing record.\n def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None:\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.local_path = local_path\n session.commit()\n finally:\n session.close()\n except Exception as e:\n log.explore(\"Failed to update repo local path\", error=str(e))\n # [/DEF:_update_repo_local_path:Function]\n\n # [DEF:_migrate_repo_directory:Function]\n # @PURPOSE: Move legacy repository directory to target path and sync DB metadata.\n # @PRE: source_path exists.\n # @POST: Repository content available at target_path.\n # @RETURN: str\n def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str:\n source_abs = os.path.abspath(source_path)\n target_abs = os.path.abspath(target_path)\n if source_abs == target_abs:\n return source_abs\n if os.path.exists(target_abs):\n log.explore(f\"Target already exists, keeping source path: {target_abs}\", error=\"Target path exists\")\n return source_abs\n Path(target_abs).parent.mkdir(parents=True, exist_ok=True)\n try:\n os.replace(source_abs, target_abs)\n except OSError:\n shutil.move(source_abs, target_abs)\n self._update_repo_local_path(dashboard_id, target_abs)\n log.reflect(f\"Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}\")\n return target_abs\n # [/DEF:_migrate_repo_directory:Function]\n\n # [DEF:_get_repo_path:Function]\n # @PURPOSE: Resolves the local filesystem path for a dashboard's repository.\n # @PARAM: dashboard_id (int)\n # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent.\n # @PRE: dashboard_id is an integer.\n # @POST: Returns DB-local_path when present, otherwise base_path/.\n # @RETURN: str\n def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str:\n with belief_scope(\"GitService._get_repo_path\"):\n if dashboard_id is None:\n raise ValueError(\"dashboard_id cannot be None\")\n self._ensure_base_path_exists()\n fallback_key = repo_key if repo_key is not None else str(dashboard_id)\n normalized_key = self._normalize_repo_key(fallback_key)\n target_path = os.path.join(self.base_path, normalized_key)\n if not self._uses_default_base_path:\n return target_path\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n finally:\n session.close()\n if db_repo and db_repo.local_path:\n db_path = os.path.abspath(db_repo.local_path)\n if os.path.exists(db_path):\n if (\n os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path)\n and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep)\n ):\n return self._migrate_repo_directory(dashboard_id, db_path, target_path)\n return db_path\n except Exception as e:\n log.explore(\"Could not resolve local_path from DB\", error=str(e))\n legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id))\n if os.path.exists(legacy_id_path) and not os.path.exists(target_path):\n return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path)\n if os.path.exists(target_path):\n self._update_repo_local_path(dashboard_id, target_path)\n return target_path\n # [/DEF:_get_repo_path:Function]\n\n # [DEF:init_repo:Function]\n # @PURPOSE: Initialize or clone a repository for a dashboard.\n # @PARAM: dashboard_id (int), remote_url (str), pat (str), repo_key (Optional[str]).\n # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided.\n # @POST: Repository is cloned or opened at the local path.\n # @RETURN: Repo\n def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo:\n with belief_scope(\"GitService.init_repo\"):\n self._ensure_base_path_exists()\n repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id))\n Path(repo_path).parent.mkdir(parents=True, exist_ok=True)\n if pat and \"://\" in remote_url:\n proto, rest = remote_url.split(\"://\", 1)\n auth_url = f\"{proto}://oauth2:{pat}@{rest}\"\n else:\n auth_url = remote_url\n if os.path.exists(repo_path):\n log.reason(f\"Opening existing repo at {repo_path}\")\n try:\n repo = Repo(repo_path)\n except (InvalidGitRepositoryError, NoSuchPathError):\n log.explore(f\"Existing path is not a Git repository, recreating: {repo_path}\", error=\"Not a git repository\")\n stale_path = Path(repo_path)\n if stale_path.exists():\n shutil.rmtree(stale_path, ignore_errors=True)\n if stale_path.exists():\n try:\n stale_path.unlink()\n except Exception:\n pass\n repo = Repo.clone_from(auth_url, repo_path)\n self._ensure_gitflow_branches(repo, dashboard_id)\n return repo\n log.reason(f\"Cloning {remote_url} to {repo_path}\")\n repo = Repo.clone_from(auth_url, repo_path)\n self._ensure_gitflow_branches(repo, dashboard_id)\n return repo\n # [/DEF:init_repo:Function]\n\n # [DEF:delete_repo:Function]\n # @PURPOSE: Remove local repository and DB binding for a dashboard.\n # @PRE: dashboard_id is a valid integer.\n # @POST: Local path is deleted when present and GitRepository row is removed.\n def delete_repo(self, dashboard_id: int) -> None:\n with belief_scope(\"GitService.delete_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n removed_files = False\n if os.path.exists(repo_path):\n if os.path.isdir(repo_path):\n shutil.rmtree(repo_path)\n else:\n os.remove(repo_path)\n removed_files = True\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n session.delete(db_repo)\n session.commit()\n return\n if removed_files:\n return\n raise HTTPException(\n status_code=404,\n detail=f\"Repository for dashboard {dashboard_id} not found\",\n )\n except HTTPException:\n session.rollback()\n raise\n except Exception as e:\n session.rollback()\n log.explore(f\"Failed to delete repository for dashboard {dashboard_id}\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Failed to delete repository: {str(e)}\")\n finally:\n session.close()\n # [/DEF:delete_repo:Function]\n\n # [DEF:get_repo:Function]\n # @PURPOSE: Get Repo object for a dashboard.\n # @PRE: Repository must exist on disk for the given dashboard_id.\n # @POST: Returns a GitPython Repo instance for the dashboard.\n # @RETURN: Repo\n def get_repo(self, dashboard_id: int) -> Repo:\n with belief_scope(\"GitService.get_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n log.explore(f\"Repository for dashboard {dashboard_id} does not exist\", error=\"Repository not found\")\n raise HTTPException(status_code=404, detail=f\"Repository for dashboard {dashboard_id} not found\")\n try:\n return Repo(repo_path)\n except Exception as e:\n log.explore(f\"Failed to open repository at {repo_path}\", error=str(e))\n raise HTTPException(status_code=500, detail=\"Failed to open local Git repository\")\n # [/DEF:get_repo:Function]\n\n # [DEF:configure_identity:Function]\n # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations.\n # @PRE: dashboard_id repository exists; git_username/git_email may be empty.\n # @POST: Repository config has user.name and user.email when both identity values are provided.\n def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None:\n with belief_scope(\"GitService.configure_identity\"):\n normalized_username = str(git_username or \"\").strip()\n normalized_email = str(git_email or \"\").strip()\n if not normalized_username or not normalized_email:\n return\n repo = self.get_repo(dashboard_id)\n try:\n with repo.config_writer(config_level=\"repository\") as config_writer:\n config_writer.set_value(\"user\", \"name\", normalized_username)\n config_writer.set_value(\"user\", \"email\", normalized_email)\n log.reason(f\"Applied repository-local git identity for dashboard {dashboard_id}\")\n except Exception as e:\n log.explore(\"Failed to configure git identity\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Failed to configure git identity: {str(e)}\")\n # [/DEF:configure_identity:Function]\n# [/DEF:GitServiceBase:Class]\n# [/DEF:GitServiceBase:Module]\n" }, { "contract_id": "GitService_init", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 29, - "end_line": 41, + "start_line": 32, + "end_line": 44, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -83832,8 +84841,8 @@ "contract_id": "_ensure_base_path_exists", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 43, - "end_line": 58, + "start_line": 46, + "end_line": 59, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -83863,14 +84872,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_ensure_base_path_exists:Function]\n # @PURPOSE: Ensure the repositories root directory exists and is a directory.\n # @PRE: self.base_path is resolved to filesystem path.\n # @POST: self.base_path exists as directory or raises ValueError.\n def _ensure_base_path_exists(self) -> None:\n base = Path(self.base_path)\n if base.exists() and not base.is_dir():\n raise ValueError(f\"Git repositories base path is not a directory: {self.base_path}\")\n try:\n base.mkdir(parents=True, exist_ok=True)\n except (PermissionError, OSError) as e:\n logger.warning(\n f\"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}\"\n )\n raise ValueError(f\"Cannot create Git repositories base path: {self.base_path}. {e}\")\n # [/DEF:_ensure_base_path_exists:Function]\n" + "body": " # [DEF:_ensure_base_path_exists:Function]\n # @PURPOSE: Ensure the repositories root directory exists and is a directory.\n # @PRE: self.base_path is resolved to filesystem path.\n # @POST: self.base_path exists as directory or raises ValueError.\n def _ensure_base_path_exists(self) -> None:\n base = Path(self.base_path)\n if base.exists() and not base.is_dir():\n raise ValueError(f\"Git repositories base path is not a directory: {self.base_path}\")\n try:\n base.mkdir(parents=True, exist_ok=True)\n except (PermissionError, OSError) as e:\n log.explore(\"Cannot create Git repositories base path\", payload={\"base_path\": self.base_path}, error=str(e))\n raise ValueError(f\"Cannot create Git repositories base path: {self.base_path}. {e}\")\n # [/DEF:_ensure_base_path_exists:Function]\n" }, { "contract_id": "_resolve_base_path", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 60, - "end_line": 93, + "start_line": 61, + "end_line": 94, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -83907,14 +84916,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_resolve_base_path:Function]\n # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings.\n # @PRE: base_path is a string path.\n # @POST: Returns absolute path for Git repositories root.\n # @RETURN: str\n def _resolve_base_path(self, base_path: str) -> str:\n backend_root = Path(__file__).parents[3]\n fallback_path = str((backend_root / base_path).resolve())\n if base_path != \"git_repos\":\n return fallback_path\n try:\n session = SessionLocal()\n try:\n config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n finally:\n session.close()\n payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {}\n storage_cfg = payload.get(\"settings\", {}).get(\"storage\", {}) if isinstance(payload, dict) else {}\n root_path = str(storage_cfg.get(\"root_path\", \"\")).strip()\n repo_path = str(storage_cfg.get(\"repo_path\", \"\")).strip()\n if not root_path:\n return fallback_path\n project_root = Path(__file__).parents[4]\n root = Path(root_path)\n if not root.is_absolute():\n root = (project_root / root).resolve()\n repo_root = Path(repo_path) if repo_path else Path(\"repositorys\")\n if repo_root.is_absolute():\n return str(repo_root.resolve())\n return str((root / repo_root).resolve())\n except Exception as e:\n logger.warning(f\"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}\")\n return fallback_path\n # [/DEF:_resolve_base_path:Function]\n" + "body": " # [DEF:_resolve_base_path:Function]\n # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings.\n # @PRE: base_path is a string path.\n # @POST: Returns absolute path for Git repositories root.\n # @RETURN: str\n def _resolve_base_path(self, base_path: str) -> str:\n backend_root = Path(__file__).parents[3]\n fallback_path = str((backend_root / base_path).resolve())\n if base_path != \"git_repos\":\n return fallback_path\n try:\n session = SessionLocal()\n try:\n config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == \"global\").first()\n finally:\n session.close()\n payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {}\n storage_cfg = payload.get(\"settings\", {}).get(\"storage\", {}) if isinstance(payload, dict) else {}\n root_path = str(storage_cfg.get(\"root_path\", \"\")).strip()\n repo_path = str(storage_cfg.get(\"repo_path\", \"\")).strip()\n if not root_path:\n return fallback_path\n project_root = Path(__file__).parents[4]\n root = Path(root_path)\n if not root.is_absolute():\n root = (project_root / root).resolve()\n repo_root = Path(repo_path) if repo_path else Path(\"repositorys\")\n if repo_root.is_absolute():\n return str(repo_root.resolve())\n return str((root / repo_root).resolve())\n except Exception as e:\n log.explore(\"Falling back to default base path\", error=str(e))\n return fallback_path\n # [/DEF:_resolve_base_path:Function]\n" }, { "contract_id": "_normalize_repo_key", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 95, - "end_line": 104, + "start_line": 96, + "end_line": 105, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -83957,8 +84966,8 @@ "contract_id": "_update_repo_local_path", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 106, - "end_line": 126, + "start_line": 107, + "end_line": 127, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -83988,14 +84997,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_update_repo_local_path:Function]\n # @PURPOSE: Persist repository local_path in GitRepository table when record exists.\n # @PRE: dashboard_id is valid integer.\n # @POST: local_path is updated for existing record.\n def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None:\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.local_path = local_path\n session.commit()\n finally:\n session.close()\n except Exception as e:\n logger.warning(f\"[_update_repo_local_path][Coherence:Failed] {e}\")\n # [/DEF:_update_repo_local_path:Function]\n" + "body": " # [DEF:_update_repo_local_path:Function]\n # @PURPOSE: Persist repository local_path in GitRepository table when record exists.\n # @PRE: dashboard_id is valid integer.\n # @POST: local_path is updated for existing record.\n def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None:\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.local_path = local_path\n session.commit()\n finally:\n session.close()\n except Exception as e:\n log.explore(\"Failed to update repo local path\", error=str(e))\n # [/DEF:_update_repo_local_path:Function]\n" }, { "contract_id": "_migrate_repo_directory", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 128, - "end_line": 151, + "start_line": 129, + "end_line": 150, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84032,14 +85041,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_migrate_repo_directory:Function]\n # @PURPOSE: Move legacy repository directory to target path and sync DB metadata.\n # @PRE: source_path exists.\n # @POST: Repository content available at target_path.\n # @RETURN: str\n def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str:\n source_abs = os.path.abspath(source_path)\n target_abs = os.path.abspath(target_path)\n if source_abs == target_abs:\n return source_abs\n if os.path.exists(target_abs):\n logger.warning(f\"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}\")\n return source_abs\n Path(target_abs).parent.mkdir(parents=True, exist_ok=True)\n try:\n os.replace(source_abs, target_abs)\n except OSError:\n shutil.move(source_abs, target_abs)\n self._update_repo_local_path(dashboard_id, target_abs)\n logger.info(\n f\"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}\"\n )\n return target_abs\n # [/DEF:_migrate_repo_directory:Function]\n" + "body": " # [DEF:_migrate_repo_directory:Function]\n # @PURPOSE: Move legacy repository directory to target path and sync DB metadata.\n # @PRE: source_path exists.\n # @POST: Repository content available at target_path.\n # @RETURN: str\n def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str:\n source_abs = os.path.abspath(source_path)\n target_abs = os.path.abspath(target_path)\n if source_abs == target_abs:\n return source_abs\n if os.path.exists(target_abs):\n log.explore(f\"Target already exists, keeping source path: {target_abs}\", error=\"Target path exists\")\n return source_abs\n Path(target_abs).parent.mkdir(parents=True, exist_ok=True)\n try:\n os.replace(source_abs, target_abs)\n except OSError:\n shutil.move(source_abs, target_abs)\n self._update_repo_local_path(dashboard_id, target_abs)\n log.reflect(f\"Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}\")\n return target_abs\n # [/DEF:_migrate_repo_directory:Function]\n" }, { "contract_id": "_get_repo_path", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 153, - "end_line": 197, + "start_line": 152, + "end_line": 196, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84083,14 +85092,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_get_repo_path:Function]\n # @PURPOSE: Resolves the local filesystem path for a dashboard's repository.\n # @PARAM: dashboard_id (int)\n # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent.\n # @PRE: dashboard_id is an integer.\n # @POST: Returns DB-local_path when present, otherwise base_path/.\n # @RETURN: str\n def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str:\n with belief_scope(\"GitService._get_repo_path\"):\n if dashboard_id is None:\n raise ValueError(\"dashboard_id cannot be None\")\n self._ensure_base_path_exists()\n fallback_key = repo_key if repo_key is not None else str(dashboard_id)\n normalized_key = self._normalize_repo_key(fallback_key)\n target_path = os.path.join(self.base_path, normalized_key)\n if not self._uses_default_base_path:\n return target_path\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n finally:\n session.close()\n if db_repo and db_repo.local_path:\n db_path = os.path.abspath(db_repo.local_path)\n if os.path.exists(db_path):\n if (\n os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path)\n and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep)\n ):\n return self._migrate_repo_directory(dashboard_id, db_path, target_path)\n return db_path\n except Exception as e:\n logger.warning(f\"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}\")\n legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id))\n if os.path.exists(legacy_id_path) and not os.path.exists(target_path):\n return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path)\n if os.path.exists(target_path):\n self._update_repo_local_path(dashboard_id, target_path)\n return target_path\n # [/DEF:_get_repo_path:Function]\n" + "body": " # [DEF:_get_repo_path:Function]\n # @PURPOSE: Resolves the local filesystem path for a dashboard's repository.\n # @PARAM: dashboard_id (int)\n # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent.\n # @PRE: dashboard_id is an integer.\n # @POST: Returns DB-local_path when present, otherwise base_path/.\n # @RETURN: str\n def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str:\n with belief_scope(\"GitService._get_repo_path\"):\n if dashboard_id is None:\n raise ValueError(\"dashboard_id cannot be None\")\n self._ensure_base_path_exists()\n fallback_key = repo_key if repo_key is not None else str(dashboard_id)\n normalized_key = self._normalize_repo_key(fallback_key)\n target_path = os.path.join(self.base_path, normalized_key)\n if not self._uses_default_base_path:\n return target_path\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n finally:\n session.close()\n if db_repo and db_repo.local_path:\n db_path = os.path.abspath(db_repo.local_path)\n if os.path.exists(db_path):\n if (\n os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path)\n and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep)\n ):\n return self._migrate_repo_directory(dashboard_id, db_path, target_path)\n return db_path\n except Exception as e:\n log.explore(\"Could not resolve local_path from DB\", error=str(e))\n legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id))\n if os.path.exists(legacy_id_path) and not os.path.exists(target_path):\n return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path)\n if os.path.exists(target_path):\n self._update_repo_local_path(dashboard_id, target_path)\n return target_path\n # [/DEF:_get_repo_path:Function]\n" }, { "contract_id": "delete_repo", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 238, - "end_line": 278, + "start_line": 237, + "end_line": 277, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84120,14 +85129,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:delete_repo:Function]\n # @PURPOSE: Remove local repository and DB binding for a dashboard.\n # @PRE: dashboard_id is a valid integer.\n # @POST: Local path is deleted when present and GitRepository row is removed.\n def delete_repo(self, dashboard_id: int) -> None:\n with belief_scope(\"GitService.delete_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n removed_files = False\n if os.path.exists(repo_path):\n if os.path.isdir(repo_path):\n shutil.rmtree(repo_path)\n else:\n os.remove(repo_path)\n removed_files = True\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n session.delete(db_repo)\n session.commit()\n return\n if removed_files:\n return\n raise HTTPException(\n status_code=404,\n detail=f\"Repository for dashboard {dashboard_id} not found\",\n )\n except HTTPException:\n session.rollback()\n raise\n except Exception as e:\n session.rollback()\n logger.error(f\"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}\")\n raise HTTPException(status_code=500, detail=f\"Failed to delete repository: {str(e)}\")\n finally:\n session.close()\n # [/DEF:delete_repo:Function]\n" + "body": " # [DEF:delete_repo:Function]\n # @PURPOSE: Remove local repository and DB binding for a dashboard.\n # @PRE: dashboard_id is a valid integer.\n # @POST: Local path is deleted when present and GitRepository row is removed.\n def delete_repo(self, dashboard_id: int) -> None:\n with belief_scope(\"GitService.delete_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n removed_files = False\n if os.path.exists(repo_path):\n if os.path.isdir(repo_path):\n shutil.rmtree(repo_path)\n else:\n os.remove(repo_path)\n removed_files = True\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n session.delete(db_repo)\n session.commit()\n return\n if removed_files:\n return\n raise HTTPException(\n status_code=404,\n detail=f\"Repository for dashboard {dashboard_id} not found\",\n )\n except HTTPException:\n session.rollback()\n raise\n except Exception as e:\n session.rollback()\n log.explore(f\"Failed to delete repository for dashboard {dashboard_id}\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Failed to delete repository: {str(e)}\")\n finally:\n session.close()\n # [/DEF:delete_repo:Function]\n" }, { "contract_id": "get_repo", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 280, - "end_line": 296, + "start_line": 279, + "end_line": 295, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84164,14 +85173,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_repo:Function]\n # @PURPOSE: Get Repo object for a dashboard.\n # @PRE: Repository must exist on disk for the given dashboard_id.\n # @POST: Returns a GitPython Repo instance for the dashboard.\n # @RETURN: Repo\n def get_repo(self, dashboard_id: int) -> Repo:\n with belief_scope(\"GitService.get_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n logger.error(f\"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist\")\n raise HTTPException(status_code=404, detail=f\"Repository for dashboard {dashboard_id} not found\")\n try:\n return Repo(repo_path)\n except Exception as e:\n logger.error(f\"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to open local Git repository\")\n # [/DEF:get_repo:Function]\n" + "body": " # [DEF:get_repo:Function]\n # @PURPOSE: Get Repo object for a dashboard.\n # @PRE: Repository must exist on disk for the given dashboard_id.\n # @POST: Returns a GitPython Repo instance for the dashboard.\n # @RETURN: Repo\n def get_repo(self, dashboard_id: int) -> Repo:\n with belief_scope(\"GitService.get_repo\"):\n repo_path = self._get_repo_path(dashboard_id)\n if not os.path.exists(repo_path):\n log.explore(f\"Repository for dashboard {dashboard_id} does not exist\", error=\"Repository not found\")\n raise HTTPException(status_code=404, detail=f\"Repository for dashboard {dashboard_id} not found\")\n try:\n return Repo(repo_path)\n except Exception as e:\n log.explore(f\"Failed to open repository at {repo_path}\", error=str(e))\n raise HTTPException(status_code=500, detail=\"Failed to open local Git repository\")\n # [/DEF:get_repo:Function]\n" }, { "contract_id": "configure_identity", "contract_type": "Function", "file_path": "backend/src/services/git/_base.py", - "start_line": 298, - "end_line": 317, + "start_line": 297, + "end_line": 316, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84201,14 +85210,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:configure_identity:Function]\n # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations.\n # @PRE: dashboard_id repository exists; git_username/git_email may be empty.\n # @POST: Repository config has user.name and user.email when both identity values are provided.\n def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None:\n with belief_scope(\"GitService.configure_identity\"):\n normalized_username = str(git_username or \"\").strip()\n normalized_email = str(git_email or \"\").strip()\n if not normalized_username or not normalized_email:\n return\n repo = self.get_repo(dashboard_id)\n try:\n with repo.config_writer(config_level=\"repository\") as config_writer:\n config_writer.set_value(\"user\", \"name\", normalized_username)\n config_writer.set_value(\"user\", \"email\", normalized_email)\n logger.info(\"[configure_identity][Action] Applied repository-local git identity for dashboard %s\", dashboard_id)\n except Exception as e:\n logger.error(f\"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}\")\n raise HTTPException(status_code=500, detail=f\"Failed to configure git identity: {str(e)}\")\n # [/DEF:configure_identity:Function]\n" + "body": " # [DEF:configure_identity:Function]\n # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations.\n # @PRE: dashboard_id repository exists; git_username/git_email may be empty.\n # @POST: Repository config has user.name and user.email when both identity values are provided.\n def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None:\n with belief_scope(\"GitService.configure_identity\"):\n normalized_username = str(git_username or \"\").strip()\n normalized_email = str(git_email or \"\").strip()\n if not normalized_username or not normalized_email:\n return\n repo = self.get_repo(dashboard_id)\n try:\n with repo.config_writer(config_level=\"repository\") as config_writer:\n config_writer.set_value(\"user\", \"name\", normalized_username)\n config_writer.set_value(\"user\", \"email\", normalized_email)\n log.reason(f\"Applied repository-local git identity for dashboard {dashboard_id}\")\n except Exception as e:\n log.explore(\"Failed to configure git identity\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Failed to configure git identity: {str(e)}\")\n # [/DEF:configure_identity:Function]\n" }, { "contract_id": "GitServiceBranchMixin", "contract_type": "Module", "file_path": "backend/src/services/git/_branch.py", "start_line": 1, - "end_line": 192, + "end_line": 187, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -84251,14 +85260,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitServiceBranchMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, branches, commits, checkout, gitflow\n# @PURPOSE: Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes.\n# @RELATION: USED_BY -> [GitService]\n\nimport os\nfrom typing import Any, Dict, List, Optional\nfrom datetime import datetime\nfrom git import Repo\nfrom git.exc import GitCommandError\nfrom fastapi import HTTPException\nfrom src.core.logger import logger, belief_scope\n\n\n# [DEF:GitServiceBranchMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing branch and commit operations for GitService.\nclass GitServiceBranchMixin:\n # [DEF:_ensure_gitflow_branches:Function]\n # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin.\n # @PRE: repo is a valid GitPython Repo instance.\n # @POST: main, dev, preprod are available in local repository and pushed to origin when available.\n def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None:\n with belief_scope(\"GitService._ensure_gitflow_branches\"):\n required_branches = [\"main\", \"dev\", \"preprod\"]\n local_heads = {head.name: head for head in getattr(repo, \"heads\", [])}\n base_commit = None\n try:\n base_commit = repo.head.commit\n except Exception:\n base_commit = None\n if \"main\" in local_heads:\n base_commit = local_heads[\"main\"].commit\n if base_commit is None:\n logger.warning(\n f\"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits\"\n )\n return\n if \"main\" not in local_heads:\n local_heads[\"main\"] = repo.create_head(\"main\", base_commit)\n logger.info(f\"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}\")\n for branch_name in (\"dev\", \"preprod\"):\n if branch_name in local_heads:\n continue\n local_heads[branch_name] = repo.create_head(branch_name, local_heads[\"main\"].commit)\n logger.info(\n f\"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}\"\n )\n try:\n origin = repo.remote(name=\"origin\")\n except ValueError:\n logger.info(\n f\"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation\"\n )\n return\n remote_branch_names = set()\n try:\n origin.fetch()\n for ref in origin.refs:\n remote_head = getattr(ref, \"remote_head\", None)\n if remote_head:\n remote_branch_names.add(str(remote_head))\n except Exception as e:\n logger.warning(f\"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}\")\n for branch_name in required_branches:\n if branch_name in remote_branch_names:\n continue\n try:\n origin.push(refspec=f\"{branch_name}:{branch_name}\")\n logger.info(\n f\"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}\"\n )\n except Exception as e:\n logger.error(f\"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}\")\n raise HTTPException(\n status_code=500,\n detail=f\"Failed to create default branch '{branch_name}' on remote: {str(e)}\",\n )\n try:\n repo.git.checkout(\"dev\")\n logger.info(f\"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}\")\n except Exception as e:\n logger.warning(f\"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}\")\n # [/DEF:_ensure_gitflow_branches:Function]\n\n # [DEF:list_branches:Function]\n # @PURPOSE: List all branches for a dashboard's repository.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of branch metadata dictionaries.\n # @RETURN: List[dict]\n def list_branches(self, dashboard_id: int) -> List[dict]:\n with belief_scope(\"GitService.list_branches\"):\n repo = self.get_repo(dashboard_id)\n logger.info(f\"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}\")\n branches = []\n for ref in repo.refs:\n try:\n name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')\n if any(b['name'] == name for b in branches):\n continue\n branches.append({\n \"name\": name,\n \"commit_hash\": ref.commit.hexsha if hasattr(ref, 'commit') else \"0000000\",\n \"is_remote\": ref.is_remote() if hasattr(ref, 'is_remote') else False,\n \"last_updated\": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()\n })\n except Exception as e:\n logger.warning(f\"[list_branches][Action] Skipping ref {ref}: {e}\")\n try:\n active_name = repo.active_branch.name\n if not any(b['name'] == active_name for b in branches):\n branches.append({\n \"name\": active_name, \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n except Exception as e:\n logger.warning(f\"[list_branches][Action] Could not determine active branch: {e}\")\n if not branches:\n branches.append({\n \"name\": \"dev\", \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n return branches\n # [/DEF:list_branches:Function]\n\n # [DEF:create_branch:Function]\n # @PURPOSE: Create a new branch from an existing one.\n # @PARAM: name (str) - New branch name.\n # @PARAM: from_branch (str) - Source branch.\n # @PRE: Repository exists; name is valid; from_branch exists or repo is empty.\n # @POST: A new branch is created in the repository.\n def create_branch(self, dashboard_id: int, name: str, from_branch: str = \"main\"):\n with belief_scope(\"GitService.create_branch\"):\n repo = self.get_repo(dashboard_id)\n logger.info(f\"[create_branch][Action] Creating branch {name} from {from_branch}\")\n if not repo.heads and not repo.remotes:\n logger.warning(\"[create_branch][Action] Repository is empty. Creating initial commit to enable branching.\")\n readme_path = os.path.join(repo.working_dir, \"README.md\")\n if not os.path.exists(readme_path):\n with open(readme_path, \"w\") as f:\n f.write(f\"# Dashboard {dashboard_id}\\nGit repository for Superset dashboard integration.\")\n repo.index.add([\"README.md\"])\n repo.index.commit(\"Initial commit\")\n try:\n repo.commit(from_branch)\n except Exception:\n logger.warning(f\"[create_branch][Action] Source branch {from_branch} not found, using HEAD\")\n from_branch = repo.head\n try:\n new_branch = repo.create_head(name, from_branch)\n return new_branch\n except Exception as e:\n logger.error(f\"[create_branch][Coherence:Failed] {e}\")\n raise\n # [/DEF:create_branch:Function]\n\n # [DEF:checkout_branch:Function]\n # @PURPOSE: Switch to a specific branch.\n # @PRE: Repository exists and the specified branch name exists.\n # @POST: The repository working directory is updated to the specified branch.\n def checkout_branch(self, dashboard_id: int, name: str):\n with belief_scope(\"GitService.checkout_branch\"):\n repo = self.get_repo(dashboard_id)\n logger.info(f\"[checkout_branch][Action] Checking out branch {name}\")\n repo.git.checkout(name)\n # [/DEF:checkout_branch:Function]\n\n # [DEF:commit_changes:Function]\n # @PURPOSE: Stage and commit changes.\n # @PARAM: message (str) - Commit message.\n # @PARAM: files (List[str]) - Optional list of specific files to stage.\n # @PRE: Repository exists and has changes (dirty) or files are specified.\n # @POST: Changes are staged and a new commit is created.\n def commit_changes(self, dashboard_id: int, message: str, files: List[str] = None):\n with belief_scope(\"GitService.commit_changes\"):\n repo = self.get_repo(dashboard_id)\n if not repo.is_dirty(untracked_files=True) and not files:\n logger.info(f\"[commit_changes][Action] No changes to commit for dashboard {dashboard_id}\")\n return\n if files:\n logger.info(f\"[commit_changes][Action] Staging files: {files}\")\n repo.index.add(files)\n else:\n logger.info(\"[commit_changes][Action] Staging all changes\")\n repo.git.add(A=True)\n repo.index.commit(message)\n logger.info(f\"[commit_changes][Coherence:OK] Committed changes with message: {message}\")\n # [/DEF:commit_changes:Function]\n# [/DEF:GitServiceBranchMixin:Class]\n# [/DEF:GitServiceBranchMixin:Module]\n" + "body": "# [DEF:GitServiceBranchMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, branches, commits, checkout, gitflow\n# @PURPOSE: Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes.\n# @RELATION: USED_BY -> [GitService]\n\nimport os\nfrom typing import Any, Dict, List, Optional\nfrom datetime import datetime\nfrom git import Repo\nfrom git.exc import GitCommandError\nfrom fastapi import HTTPException\nfrom src.core.logger import belief_scope\nfrom src.core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"GitBranch\")\n\n\n# [DEF:GitServiceBranchMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing branch and commit operations for GitService.\nclass GitServiceBranchMixin:\n # [DEF:_ensure_gitflow_branches:Function]\n # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin.\n # @PRE: repo is a valid GitPython Repo instance.\n # @POST: main, dev, preprod are available in local repository and pushed to origin when available.\n def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None:\n with belief_scope(\"GitService._ensure_gitflow_branches\"):\n required_branches = [\"main\", \"dev\", \"preprod\"]\n local_heads = {head.name: head for head in getattr(repo, \"heads\", [])}\n base_commit = None\n try:\n base_commit = repo.head.commit\n except Exception:\n base_commit = None\n if \"main\" in local_heads:\n base_commit = local_heads[\"main\"].commit\n if base_commit is None:\n log.explore(\"Branch bootstrap skipped - no commits\", payload={\"dashboard_id\": dashboard_id}, error=\"Repository has no initial commit to branch from\")\n return\n if \"main\" not in local_heads:\n local_heads[\"main\"] = repo.create_head(\"main\", base_commit)\n log.reason(\"Created local branch main\", payload={\"dashboard_id\": dashboard_id})\n for branch_name in (\"dev\", \"preprod\"):\n if branch_name in local_heads:\n continue\n local_heads[branch_name] = repo.create_head(branch_name, local_heads[\"main\"].commit)\n log.reason(\"Created local branch\", payload={\"branch_name\": branch_name, \"dashboard_id\": dashboard_id})\n try:\n origin = repo.remote(name=\"origin\")\n except ValueError:\n log.reason(\"Remote origin not configured; skipping remote branch creation\", payload={\"dashboard_id\": dashboard_id})\n return\n remote_branch_names = set()\n try:\n origin.fetch()\n for ref in origin.refs:\n remote_head = getattr(ref, \"remote_head\", None)\n if remote_head:\n remote_branch_names.add(str(remote_head))\n except Exception as e:\n log.explore(\"Failed to fetch origin refs\", error=str(e))\n for branch_name in required_branches:\n if branch_name in remote_branch_names:\n continue\n try:\n origin.push(refspec=f\"{branch_name}:{branch_name}\")\n log.reason(\"Pushed branch to origin\", payload={\"branch_name\": branch_name, \"dashboard_id\": dashboard_id})\n except Exception as e:\n log.explore(\"Failed to push branch to origin\", error=str(e))\n raise HTTPException(\n status_code=500,\n detail=f\"Failed to create default branch '{branch_name}' on remote: {str(e)}\",\n )\n try:\n repo.git.checkout(\"dev\")\n log.reason(\"Checked out default branch dev\", payload={\"dashboard_id\": dashboard_id})\n except Exception as e:\n log.explore(\"Could not checkout dev branch\", payload={\"dashboard_id\": dashboard_id}, error=str(e))\n # [/DEF:_ensure_gitflow_branches:Function]\n\n # [DEF:list_branches:Function]\n # @PURPOSE: List all branches for a dashboard's repository.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of branch metadata dictionaries.\n # @RETURN: List[dict]\n def list_branches(self, dashboard_id: int) -> List[dict]:\n with belief_scope(\"GitService.list_branches\"):\n repo = self.get_repo(dashboard_id)\n log.reason(\"Listing branches\", payload={\"dashboard_id\": dashboard_id})\n branches = []\n for ref in repo.refs:\n try:\n name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')\n if any(b['name'] == name for b in branches):\n continue\n branches.append({\n \"name\": name,\n \"commit_hash\": ref.commit.hexsha if hasattr(ref, 'commit') else \"0000000\",\n \"is_remote\": ref.is_remote() if hasattr(ref, 'is_remote') else False,\n \"last_updated\": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()\n })\n except Exception as e:\n log.explore(\"Skipping ref\", payload={\"ref\": str(ref)}, error=str(e))\n try:\n active_name = repo.active_branch.name\n if not any(b['name'] == active_name for b in branches):\n branches.append({\n \"name\": active_name, \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n except Exception as e:\n log.explore(\"Could not determine active branch\", error=str(e))\n if not branches:\n branches.append({\n \"name\": \"dev\", \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n return branches\n # [/DEF:list_branches:Function]\n\n # [DEF:create_branch:Function]\n # @PURPOSE: Create a new branch from an existing one.\n # @PARAM: name (str) - New branch name.\n # @PARAM: from_branch (str) - Source branch.\n # @PRE: Repository exists; name is valid; from_branch exists or repo is empty.\n # @POST: A new branch is created in the repository.\n def create_branch(self, dashboard_id: int, name: str, from_branch: str = \"main\"):\n with belief_scope(\"GitService.create_branch\"):\n repo = self.get_repo(dashboard_id)\n log.reason(\"Creating branch\", payload={\"name\": name, \"from_branch\": from_branch})\n if not repo.heads and not repo.remotes:\n log.explore(\"Repository is empty; creating initial commit to enable branching\", error=\"No branches or remotes exist; initializing from scratch\")\n readme_path = os.path.join(repo.working_dir, \"README.md\")\n if not os.path.exists(readme_path):\n with open(readme_path, \"w\") as f:\n f.write(f\"# Dashboard {dashboard_id}\\nGit repository for Superset dashboard integration.\")\n repo.index.add([\"README.md\"])\n repo.index.commit(\"Initial commit\")\n try:\n repo.commit(from_branch)\n except Exception:\n log.explore(\"Source branch not found, using HEAD\", payload={\"from_branch\": from_branch}, error=f\"Branch '{from_branch}' does not exist, falling back to HEAD\")\n from_branch = repo.head\n try:\n new_branch = repo.create_head(name, from_branch)\n return new_branch\n except Exception as e:\n log.explore(\"Failed to create branch\", error=str(e))\n raise\n # [/DEF:create_branch:Function]\n\n # [DEF:checkout_branch:Function]\n # @PURPOSE: Switch to a specific branch.\n # @PRE: Repository exists and the specified branch name exists.\n # @POST: The repository working directory is updated to the specified branch.\n def checkout_branch(self, dashboard_id: int, name: str):\n with belief_scope(\"GitService.checkout_branch\"):\n repo = self.get_repo(dashboard_id)\n log.reason(\"Checking out branch\", payload={\"name\": name})\n repo.git.checkout(name)\n # [/DEF:checkout_branch:Function]\n\n # [DEF:commit_changes:Function]\n # @PURPOSE: Stage and commit changes.\n # @PARAM: message (str) - Commit message.\n # @PARAM: files (List[str]) - Optional list of specific files to stage.\n # @PRE: Repository exists and has changes (dirty) or files are specified.\n # @POST: Changes are staged and a new commit is created.\n def commit_changes(self, dashboard_id: int, message: str, files: List[str] = None):\n with belief_scope(\"GitService.commit_changes\"):\n repo = self.get_repo(dashboard_id)\n if not repo.is_dirty(untracked_files=True) and not files:\n log.reason(\"No changes to commit\", payload={\"dashboard_id\": dashboard_id})\n return\n if files:\n log.reason(\"Staging files\", payload={\"files\": files})\n repo.index.add(files)\n else:\n log.reason(\"Staging all changes\")\n repo.git.add(A=True)\n repo.index.commit(message)\n log.reflect(\"Committed changes\", payload={\"message\": message})\n # [/DEF:commit_changes:Function]\n# [/DEF:GitServiceBranchMixin:Class]\n# [/DEF:GitServiceBranchMixin:Module]\n" }, { "contract_id": "_ensure_gitflow_branches", "contract_type": "Function", "file_path": "backend/src/services/git/_branch.py", - "start_line": 21, - "end_line": 86, + "start_line": 24, + "end_line": 81, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84288,14 +85297,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_ensure_gitflow_branches:Function]\n # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin.\n # @PRE: repo is a valid GitPython Repo instance.\n # @POST: main, dev, preprod are available in local repository and pushed to origin when available.\n def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None:\n with belief_scope(\"GitService._ensure_gitflow_branches\"):\n required_branches = [\"main\", \"dev\", \"preprod\"]\n local_heads = {head.name: head for head in getattr(repo, \"heads\", [])}\n base_commit = None\n try:\n base_commit = repo.head.commit\n except Exception:\n base_commit = None\n if \"main\" in local_heads:\n base_commit = local_heads[\"main\"].commit\n if base_commit is None:\n logger.warning(\n f\"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits\"\n )\n return\n if \"main\" not in local_heads:\n local_heads[\"main\"] = repo.create_head(\"main\", base_commit)\n logger.info(f\"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}\")\n for branch_name in (\"dev\", \"preprod\"):\n if branch_name in local_heads:\n continue\n local_heads[branch_name] = repo.create_head(branch_name, local_heads[\"main\"].commit)\n logger.info(\n f\"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}\"\n )\n try:\n origin = repo.remote(name=\"origin\")\n except ValueError:\n logger.info(\n f\"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation\"\n )\n return\n remote_branch_names = set()\n try:\n origin.fetch()\n for ref in origin.refs:\n remote_head = getattr(ref, \"remote_head\", None)\n if remote_head:\n remote_branch_names.add(str(remote_head))\n except Exception as e:\n logger.warning(f\"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}\")\n for branch_name in required_branches:\n if branch_name in remote_branch_names:\n continue\n try:\n origin.push(refspec=f\"{branch_name}:{branch_name}\")\n logger.info(\n f\"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}\"\n )\n except Exception as e:\n logger.error(f\"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}\")\n raise HTTPException(\n status_code=500,\n detail=f\"Failed to create default branch '{branch_name}' on remote: {str(e)}\",\n )\n try:\n repo.git.checkout(\"dev\")\n logger.info(f\"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}\")\n except Exception as e:\n logger.warning(f\"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}\")\n # [/DEF:_ensure_gitflow_branches:Function]\n" + "body": " # [DEF:_ensure_gitflow_branches:Function]\n # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin.\n # @PRE: repo is a valid GitPython Repo instance.\n # @POST: main, dev, preprod are available in local repository and pushed to origin when available.\n def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None:\n with belief_scope(\"GitService._ensure_gitflow_branches\"):\n required_branches = [\"main\", \"dev\", \"preprod\"]\n local_heads = {head.name: head for head in getattr(repo, \"heads\", [])}\n base_commit = None\n try:\n base_commit = repo.head.commit\n except Exception:\n base_commit = None\n if \"main\" in local_heads:\n base_commit = local_heads[\"main\"].commit\n if base_commit is None:\n log.explore(\"Branch bootstrap skipped - no commits\", payload={\"dashboard_id\": dashboard_id}, error=\"Repository has no initial commit to branch from\")\n return\n if \"main\" not in local_heads:\n local_heads[\"main\"] = repo.create_head(\"main\", base_commit)\n log.reason(\"Created local branch main\", payload={\"dashboard_id\": dashboard_id})\n for branch_name in (\"dev\", \"preprod\"):\n if branch_name in local_heads:\n continue\n local_heads[branch_name] = repo.create_head(branch_name, local_heads[\"main\"].commit)\n log.reason(\"Created local branch\", payload={\"branch_name\": branch_name, \"dashboard_id\": dashboard_id})\n try:\n origin = repo.remote(name=\"origin\")\n except ValueError:\n log.reason(\"Remote origin not configured; skipping remote branch creation\", payload={\"dashboard_id\": dashboard_id})\n return\n remote_branch_names = set()\n try:\n origin.fetch()\n for ref in origin.refs:\n remote_head = getattr(ref, \"remote_head\", None)\n if remote_head:\n remote_branch_names.add(str(remote_head))\n except Exception as e:\n log.explore(\"Failed to fetch origin refs\", error=str(e))\n for branch_name in required_branches:\n if branch_name in remote_branch_names:\n continue\n try:\n origin.push(refspec=f\"{branch_name}:{branch_name}\")\n log.reason(\"Pushed branch to origin\", payload={\"branch_name\": branch_name, \"dashboard_id\": dashboard_id})\n except Exception as e:\n log.explore(\"Failed to push branch to origin\", error=str(e))\n raise HTTPException(\n status_code=500,\n detail=f\"Failed to create default branch '{branch_name}' on remote: {str(e)}\",\n )\n try:\n repo.git.checkout(\"dev\")\n log.reason(\"Checked out default branch dev\", payload={\"dashboard_id\": dashboard_id})\n except Exception as e:\n log.explore(\"Could not checkout dev branch\", payload={\"dashboard_id\": dashboard_id}, error=str(e))\n # [/DEF:_ensure_gitflow_branches:Function]\n" }, { "contract_id": "list_branches", "contract_type": "Function", "file_path": "backend/src/services/git/_branch.py", - "start_line": 88, - "end_line": 126, + "start_line": 83, + "end_line": 121, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84332,7 +85341,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:list_branches:Function]\n # @PURPOSE: List all branches for a dashboard's repository.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of branch metadata dictionaries.\n # @RETURN: List[dict]\n def list_branches(self, dashboard_id: int) -> List[dict]:\n with belief_scope(\"GitService.list_branches\"):\n repo = self.get_repo(dashboard_id)\n logger.info(f\"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}\")\n branches = []\n for ref in repo.refs:\n try:\n name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')\n if any(b['name'] == name for b in branches):\n continue\n branches.append({\n \"name\": name,\n \"commit_hash\": ref.commit.hexsha if hasattr(ref, 'commit') else \"0000000\",\n \"is_remote\": ref.is_remote() if hasattr(ref, 'is_remote') else False,\n \"last_updated\": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()\n })\n except Exception as e:\n logger.warning(f\"[list_branches][Action] Skipping ref {ref}: {e}\")\n try:\n active_name = repo.active_branch.name\n if not any(b['name'] == active_name for b in branches):\n branches.append({\n \"name\": active_name, \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n except Exception as e:\n logger.warning(f\"[list_branches][Action] Could not determine active branch: {e}\")\n if not branches:\n branches.append({\n \"name\": \"dev\", \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n return branches\n # [/DEF:list_branches:Function]\n" + "body": " # [DEF:list_branches:Function]\n # @PURPOSE: List all branches for a dashboard's repository.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of branch metadata dictionaries.\n # @RETURN: List[dict]\n def list_branches(self, dashboard_id: int) -> List[dict]:\n with belief_scope(\"GitService.list_branches\"):\n repo = self.get_repo(dashboard_id)\n log.reason(\"Listing branches\", payload={\"dashboard_id\": dashboard_id})\n branches = []\n for ref in repo.refs:\n try:\n name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')\n if any(b['name'] == name for b in branches):\n continue\n branches.append({\n \"name\": name,\n \"commit_hash\": ref.commit.hexsha if hasattr(ref, 'commit') else \"0000000\",\n \"is_remote\": ref.is_remote() if hasattr(ref, 'is_remote') else False,\n \"last_updated\": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()\n })\n except Exception as e:\n log.explore(\"Skipping ref\", payload={\"ref\": str(ref)}, error=str(e))\n try:\n active_name = repo.active_branch.name\n if not any(b['name'] == active_name for b in branches):\n branches.append({\n \"name\": active_name, \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n except Exception as e:\n log.explore(\"Could not determine active branch\", error=str(e))\n if not branches:\n branches.append({\n \"name\": \"dev\", \"commit_hash\": \"0000000\",\n \"is_remote\": False, \"last_updated\": datetime.utcnow()\n })\n return branches\n # [/DEF:list_branches:Function]\n" }, { "contract_id": "GitServiceGiteaMixin", @@ -84382,14 +85391,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitServiceGiteaMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, gitea, api, provider, connection_test\n# @PURPOSE: Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation.\n# @RELATION: USED_BY -> [GitService]\n\nimport httpx\nfrom typing import Any, Dict, List, Optional\nfrom urllib.parse import quote\nfrom fastapi import HTTPException\nfrom src.core.logger import logger, belief_scope\nfrom src.models.git import GitProvider\n\n\n# [DEF:GitServiceGiteaMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing Gitea API operations for GitService.\nclass GitServiceGiteaMixin:\n # [DEF:test_connection:Function]\n # @PURPOSE: Test connection to Git provider using PAT.\n # @PARAM: provider (GitProvider), url (str), pat (str)\n # @PRE: provider is valid; url is a valid HTTP(S) URL; pat is provided.\n # @POST: Returns True if connection to the provider's API succeeds.\n # @RETURN: bool\n async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool:\n with belief_scope(\"GitService.test_connection\"):\n if \".local\" in url or \"localhost\" in url:\n logger.info(\"[test_connection][Action] Local/Offline mode detected for URL\")\n return True\n if not url.startswith(('http://', 'https://')):\n logger.error(f\"[test_connection][Coherence:Failed] Invalid URL protocol: {url}\")\n return False\n if not pat or not pat.strip():\n logger.error(\"[test_connection][Coherence:Failed] Git PAT is missing or empty\")\n return False\n pat = pat.strip()\n try:\n async with httpx.AsyncClient() as client:\n if provider == GitProvider.GITHUB:\n headers = {\"Authorization\": f\"token {pat}\"}\n api_url = \"https://api.github.com/user\" if \"github.com\" in url else f\"{url.rstrip('/')}/api/v3/user\"\n resp = await client.get(api_url, headers=headers)\n elif provider == GitProvider.GITLAB:\n headers = {\"PRIVATE-TOKEN\": pat}\n api_url = f\"{url.rstrip('/')}/api/v4/user\"\n resp = await client.get(api_url, headers=headers)\n elif provider == GitProvider.GITEA:\n headers = {\"Authorization\": f\"token {pat}\"}\n api_url = f\"{url.rstrip('/')}/api/v1/user\"\n resp = await client.get(api_url, headers=headers)\n else:\n return False\n if resp.status_code != 200:\n logger.error(f\"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}\")\n return resp.status_code == 200\n except Exception as e:\n logger.error(f\"[test_connection][Coherence:Failed] Error testing git connection: {e}\")\n return False\n # [/DEF:test_connection:Function]\n\n # [DEF:_gitea_headers:Function]\n # @PURPOSE: Build Gitea API authorization headers.\n # @PRE: pat is provided.\n # @POST: Returns headers with token auth.\n # @RETURN: Dict[str, str]\n def _gitea_headers(self, pat: str) -> Dict[str, str]:\n token = (pat or \"\").strip()\n if not token:\n raise HTTPException(status_code=400, detail=\"Git PAT is required for Gitea operations\")\n return {\n \"Authorization\": f\"token {token}\",\n \"Content-Type\": \"application/json\",\n \"Accept\": \"application/json\",\n }\n # [/DEF:_gitea_headers:Function]\n\n # [DEF:_gitea_request:Function]\n # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping.\n # @PRE: method and endpoint are valid.\n # @POST: Returns decoded JSON payload.\n # @RETURN: Any\n async def _gitea_request(\n self, method: str, server_url: str, pat: str, endpoint: str,\n payload: Optional[Dict[str, Any]] = None,\n ) -> Any:\n base_url = self._normalize_git_server_url(server_url)\n url = f\"{base_url}/api/v1{endpoint}\"\n headers = self._gitea_headers(pat)\n try:\n async with httpx.AsyncClient(timeout=20.0) as client:\n response = await client.request(method=method, url=url, headers=headers, json=payload)\n except Exception as e:\n logger.error(f\"[gitea_request][Coherence:Failed] Network error: {e}\")\n raise HTTPException(status_code=503, detail=f\"Gitea API is unavailable: {str(e)}\")\n if response.status_code >= 400:\n detail = response.text\n try:\n parsed = response.json()\n detail = parsed.get(\"message\") or parsed.get(\"error\") or detail\n except Exception:\n pass\n logger.error(f\"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}\")\n raise HTTPException(status_code=response.status_code, detail=f\"Gitea API error: {detail}\")\n if response.status_code == 204:\n return None\n return response.json()\n # [/DEF:_gitea_request:Function]\n\n # [DEF:get_gitea_current_user:Function]\n # @PURPOSE: Resolve current Gitea user for PAT.\n # @PRE: server_url and pat are valid.\n # @POST: Returns current username.\n # @RETURN: str\n async def get_gitea_current_user(self, server_url: str, pat: str) -> str:\n payload = await self._gitea_request(\"GET\", server_url, pat, \"/user\")\n username = payload.get(\"login\") or payload.get(\"username\")\n if not username:\n raise HTTPException(status_code=500, detail=\"Failed to resolve Gitea username\")\n return str(username)\n # [/DEF:get_gitea_current_user:Function]\n\n # [DEF:list_gitea_repositories:Function]\n # @PURPOSE: List repositories visible to authenticated Gitea user.\n # @PRE: server_url and pat are valid.\n # @POST: Returns repository list from Gitea.\n # @RETURN: List[dict]\n async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]:\n payload = await self._gitea_request(\"GET\", server_url, pat, \"/user/repos?limit=100&page=1\")\n if not isinstance(payload, list):\n return []\n return payload\n # [/DEF:list_gitea_repositories:Function]\n\n # [DEF:create_gitea_repository:Function]\n # @PURPOSE: Create repository in Gitea for authenticated user.\n # @PRE: name is non-empty and PAT has repo creation permission.\n # @POST: Returns created repository payload.\n # @RETURN: dict\n async def create_gitea_repository(\n self, server_url: str, pat: str, name: str, private: bool = True,\n description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = \"main\",\n ) -> Dict[str, Any]:\n payload: Dict[str, Any] = {\"name\": name, \"private\": bool(private), \"auto_init\": bool(auto_init)}\n if description:\n payload[\"description\"] = description\n if default_branch:\n payload[\"default_branch\"] = default_branch\n created = await self._gitea_request(\"POST\", server_url, pat, \"/user/repos\", payload=payload)\n if not isinstance(created, dict):\n raise HTTPException(status_code=500, detail=\"Unexpected Gitea response while creating repository\")\n return created\n # [/DEF:create_gitea_repository:Function]\n\n # [DEF:delete_gitea_repository:Function]\n # @PURPOSE: Delete repository in Gitea.\n # @PRE: owner and repo_name are non-empty.\n # @POST: Repository deleted on Gitea server.\n async def delete_gitea_repository(self, server_url: str, pat: str, owner: str, repo_name: str) -> None:\n if not owner or not repo_name:\n raise HTTPException(status_code=400, detail=\"owner and repo_name are required\")\n await self._gitea_request(\"DELETE\", server_url, pat, f\"/repos/{owner}/{repo_name}\")\n # [/DEF:delete_gitea_repository:Function]\n\n # [DEF:_gitea_branch_exists:Function]\n # @PURPOSE: Check whether a branch exists in Gitea repository.\n # @PRE: owner/repo/branch are non-empty.\n # @POST: Returns True when branch exists, False when 404.\n # @RETURN: bool\n async def _gitea_branch_exists(self, server_url: str, pat: str, owner: str, repo: str, branch: str) -> bool:\n if not owner or not repo or not branch:\n return False\n endpoint = f\"/repos/{owner}/{repo}/branches/{quote(branch, safe='')}\"\n try:\n await self._gitea_request(\"GET\", server_url, pat, endpoint)\n return True\n except HTTPException as exc:\n if exc.status_code == 404:\n return False\n raise\n # [/DEF:_gitea_branch_exists:Function]\n\n # [DEF:_build_gitea_pr_404_detail:Function]\n # @PURPOSE: Build actionable error detail for Gitea PR 404 responses.\n # @PRE: owner/repo/from_branch/to_branch are provided.\n # @POST: Returns specific branch-missing message when detected.\n # @RETURN: Optional[str]\n async def _build_gitea_pr_404_detail(\n self, server_url: str, pat: str, owner: str, repo: str, from_branch: str, to_branch: str,\n ) -> Optional[str]:\n source_exists = await self._gitea_branch_exists(\n server_url=server_url, pat=pat, owner=owner, repo=repo, branch=from_branch,\n )\n target_exists = await self._gitea_branch_exists(\n server_url=server_url, pat=pat, owner=owner, repo=repo, branch=to_branch,\n )\n if not source_exists:\n return f\"Gitea branch not found: source branch '{from_branch}' in {owner}/{repo}\"\n if not target_exists:\n return f\"Gitea branch not found: target branch '{to_branch}' in {owner}/{repo}\"\n return None\n # [/DEF:_build_gitea_pr_404_detail:Function]\n\n # [DEF:create_gitea_pull_request:Function]\n # @PURPOSE: Create pull request in Gitea.\n # @PRE: Config and remote URL are valid.\n # @POST: Returns normalized PR metadata.\n # @RETURN: Dict[str, Any]\n async def create_gitea_pull_request(\n self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,\n title: str, description: Optional[str] = None,\n ) -> Dict[str, Any]:\n identity = self._parse_remote_repo_identity(remote_url)\n req_payload = {\"title\": title, \"head\": from_branch, \"base\": to_branch, \"body\": description or \"\"}\n endpoint = f\"/repos/{identity['owner']}/{identity['repo']}/pulls\"\n active_server_url = server_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as exc:\n fallback_url = self._derive_server_url_from_remote(remote_url)\n normalized_primary = self._normalize_git_server_url(server_url)\n should_retry_with_fallback = (\n exc.status_code == 404 and fallback_url and fallback_url != normalized_primary\n )\n if should_retry_with_fallback:\n logger.warning(\n \"[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s\",\n fallback_url,\n )\n active_server_url = fallback_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as retry_exc:\n if retry_exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n else:\n if exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n if not isinstance(data, dict):\n raise HTTPException(status_code=500, detail=\"Unexpected Gitea response while creating pull request\")\n return {\n \"id\": data.get(\"number\") or data.get(\"id\"),\n \"url\": data.get(\"html_url\") or data.get(\"url\"),\n \"status\": data.get(\"state\") or \"open\",\n }\n # [/DEF:create_gitea_pull_request:Function]\n# [/DEF:GitServiceGiteaMixin:Class]\n# [/DEF:GitServiceGiteaMixin:Module]\n" + "body": "# [DEF:GitServiceGiteaMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, gitea, api, provider, connection_test\n# @PURPOSE: Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation.\n# @RELATION: USED_BY -> [GitService]\n\nimport httpx\nfrom typing import Any, Dict, List, Optional\nfrom urllib.parse import quote\nfrom fastapi import HTTPException\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\n\nlog = MarkerLogger(\"GitGitea\")\nfrom src.models.git import GitProvider\n\n\n# [DEF:GitServiceGiteaMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing Gitea API operations for GitService.\nclass GitServiceGiteaMixin:\n # [DEF:test_connection:Function]\n # @PURPOSE: Test connection to Git provider using PAT.\n # @PARAM: provider (GitProvider), url (str), pat (str)\n # @PRE: provider is valid; url is a valid HTTP(S) URL; pat is provided.\n # @POST: Returns True if connection to the provider's API succeeds.\n # @RETURN: bool\n async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool:\n with belief_scope(\"GitService.test_connection\"):\n if \".local\" in url or \"localhost\" in url:\n log.reason(\"Local/Offline mode detected for URL\")\n return True\n if not url.startswith(('http://', 'https://')):\n log.explore(f\"Invalid URL protocol: {url}\", error=\"Invalid URL protocol\")\n return False\n if not pat or not pat.strip():\n log.explore(\"Git PAT is missing or empty\", error=\"Git PAT is missing or empty\")\n return False\n pat = pat.strip()\n try:\n async with httpx.AsyncClient() as client:\n if provider == GitProvider.GITHUB:\n headers = {\"Authorization\": f\"token {pat}\"}\n api_url = \"https://api.github.com/user\" if \"github.com\" in url else f\"{url.rstrip('/')}/api/v3/user\"\n resp = await client.get(api_url, headers=headers)\n elif provider == GitProvider.GITLAB:\n headers = {\"PRIVATE-TOKEN\": pat}\n api_url = f\"{url.rstrip('/')}/api/v4/user\"\n resp = await client.get(api_url, headers=headers)\n elif provider == GitProvider.GITEA:\n headers = {\"Authorization\": f\"token {pat}\"}\n api_url = f\"{url.rstrip('/')}/api/v1/user\"\n resp = await client.get(api_url, headers=headers)\n else:\n return False\n if resp.status_code != 200:\n log.explore(f\"Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}\", error=\"Git connection test failed\")\n return resp.status_code == 200\n except Exception as e:\n log.explore(f\"Error testing git connection: {e}\", error=str(e))\n return False\n # [/DEF:test_connection:Function]\n\n # [DEF:_gitea_headers:Function]\n # @PURPOSE: Build Gitea API authorization headers.\n # @PRE: pat is provided.\n # @POST: Returns headers with token auth.\n # @RETURN: Dict[str, str]\n def _gitea_headers(self, pat: str) -> Dict[str, str]:\n token = (pat or \"\").strip()\n if not token:\n raise HTTPException(status_code=400, detail=\"Git PAT is required for Gitea operations\")\n return {\n \"Authorization\": f\"token {token}\",\n \"Content-Type\": \"application/json\",\n \"Accept\": \"application/json\",\n }\n # [/DEF:_gitea_headers:Function]\n\n # [DEF:_gitea_request:Function]\n # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping.\n # @PRE: method and endpoint are valid.\n # @POST: Returns decoded JSON payload.\n # @RETURN: Any\n async def _gitea_request(\n self, method: str, server_url: str, pat: str, endpoint: str,\n payload: Optional[Dict[str, Any]] = None,\n ) -> Any:\n base_url = self._normalize_git_server_url(server_url)\n url = f\"{base_url}/api/v1{endpoint}\"\n headers = self._gitea_headers(pat)\n try:\n async with httpx.AsyncClient(timeout=20.0) as client:\n response = await client.request(method=method, url=url, headers=headers, json=payload)\n except Exception as e:\n log.explore(f\"Network error: {e}\", error=str(e))\n raise HTTPException(status_code=503, detail=f\"Gitea API is unavailable: {str(e)}\")\n if response.status_code >= 400:\n detail = response.text\n try:\n parsed = response.json()\n detail = parsed.get(\"message\") or parsed.get(\"error\") or detail\n except Exception:\n pass\n log.explore(f\"Gitea API error ({endpoint})\", payload={\"method\": method, \"status\": response.status_code}, error=detail)\n raise HTTPException(status_code=response.status_code, detail=f\"Gitea API error: {detail}\")\n if response.status_code == 204:\n return None\n return response.json()\n # [/DEF:_gitea_request:Function]\n\n # [DEF:get_gitea_current_user:Function]\n # @PURPOSE: Resolve current Gitea user for PAT.\n # @PRE: server_url and pat are valid.\n # @POST: Returns current username.\n # @RETURN: str\n async def get_gitea_current_user(self, server_url: str, pat: str) -> str:\n payload = await self._gitea_request(\"GET\", server_url, pat, \"/user\")\n username = payload.get(\"login\") or payload.get(\"username\")\n if not username:\n raise HTTPException(status_code=500, detail=\"Failed to resolve Gitea username\")\n return str(username)\n # [/DEF:get_gitea_current_user:Function]\n\n # [DEF:list_gitea_repositories:Function]\n # @PURPOSE: List repositories visible to authenticated Gitea user.\n # @PRE: server_url and pat are valid.\n # @POST: Returns repository list from Gitea.\n # @RETURN: List[dict]\n async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]:\n payload = await self._gitea_request(\"GET\", server_url, pat, \"/user/repos?limit=100&page=1\")\n if not isinstance(payload, list):\n return []\n return payload\n # [/DEF:list_gitea_repositories:Function]\n\n # [DEF:create_gitea_repository:Function]\n # @PURPOSE: Create repository in Gitea for authenticated user.\n # @PRE: name is non-empty and PAT has repo creation permission.\n # @POST: Returns created repository payload.\n # @RETURN: dict\n async def create_gitea_repository(\n self, server_url: str, pat: str, name: str, private: bool = True,\n description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = \"main\",\n ) -> Dict[str, Any]:\n payload: Dict[str, Any] = {\"name\": name, \"private\": bool(private), \"auto_init\": bool(auto_init)}\n if description:\n payload[\"description\"] = description\n if default_branch:\n payload[\"default_branch\"] = default_branch\n created = await self._gitea_request(\"POST\", server_url, pat, \"/user/repos\", payload=payload)\n if not isinstance(created, dict):\n raise HTTPException(status_code=500, detail=\"Unexpected Gitea response while creating repository\")\n return created\n # [/DEF:create_gitea_repository:Function]\n\n # [DEF:delete_gitea_repository:Function]\n # @PURPOSE: Delete repository in Gitea.\n # @PRE: owner and repo_name are non-empty.\n # @POST: Repository deleted on Gitea server.\n async def delete_gitea_repository(self, server_url: str, pat: str, owner: str, repo_name: str) -> None:\n if not owner or not repo_name:\n raise HTTPException(status_code=400, detail=\"owner and repo_name are required\")\n await self._gitea_request(\"DELETE\", server_url, pat, f\"/repos/{owner}/{repo_name}\")\n # [/DEF:delete_gitea_repository:Function]\n\n # [DEF:_gitea_branch_exists:Function]\n # @PURPOSE: Check whether a branch exists in Gitea repository.\n # @PRE: owner/repo/branch are non-empty.\n # @POST: Returns True when branch exists, False when 404.\n # @RETURN: bool\n async def _gitea_branch_exists(self, server_url: str, pat: str, owner: str, repo: str, branch: str) -> bool:\n if not owner or not repo or not branch:\n return False\n endpoint = f\"/repos/{owner}/{repo}/branches/{quote(branch, safe='')}\"\n try:\n await self._gitea_request(\"GET\", server_url, pat, endpoint)\n return True\n except HTTPException as exc:\n if exc.status_code == 404:\n return False\n raise\n # [/DEF:_gitea_branch_exists:Function]\n\n # [DEF:_build_gitea_pr_404_detail:Function]\n # @PURPOSE: Build actionable error detail for Gitea PR 404 responses.\n # @PRE: owner/repo/from_branch/to_branch are provided.\n # @POST: Returns specific branch-missing message when detected.\n # @RETURN: Optional[str]\n async def _build_gitea_pr_404_detail(\n self, server_url: str, pat: str, owner: str, repo: str, from_branch: str, to_branch: str,\n ) -> Optional[str]:\n source_exists = await self._gitea_branch_exists(\n server_url=server_url, pat=pat, owner=owner, repo=repo, branch=from_branch,\n )\n target_exists = await self._gitea_branch_exists(\n server_url=server_url, pat=pat, owner=owner, repo=repo, branch=to_branch,\n )\n if not source_exists:\n return f\"Gitea branch not found: source branch '{from_branch}' in {owner}/{repo}\"\n if not target_exists:\n return f\"Gitea branch not found: target branch '{to_branch}' in {owner}/{repo}\"\n return None\n # [/DEF:_build_gitea_pr_404_detail:Function]\n\n # [DEF:create_gitea_pull_request:Function]\n # @PURPOSE: Create pull request in Gitea.\n # @PRE: Config and remote URL are valid.\n # @POST: Returns normalized PR metadata.\n # @RETURN: Dict[str, Any]\n async def create_gitea_pull_request(\n self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,\n title: str, description: Optional[str] = None,\n ) -> Dict[str, Any]:\n identity = self._parse_remote_repo_identity(remote_url)\n req_payload = {\"title\": title, \"head\": from_branch, \"base\": to_branch, \"body\": description or \"\"}\n endpoint = f\"/repos/{identity['owner']}/{identity['repo']}/pulls\"\n active_server_url = server_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as exc:\n fallback_url = self._derive_server_url_from_remote(remote_url)\n normalized_primary = self._normalize_git_server_url(server_url)\n should_retry_with_fallback = (\n exc.status_code == 404 and fallback_url and fallback_url != normalized_primary\n )\n if should_retry_with_fallback:\n log.explore(f\"Primary Gitea URL not found, retrying with remote host: {fallback_url}\", error=fallback_url)\n active_server_url = fallback_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as retry_exc:\n if retry_exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n else:\n if exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n if not isinstance(data, dict):\n raise HTTPException(status_code=500, detail=\"Unexpected Gitea response while creating pull request\")\n return {\n \"id\": data.get(\"number\") or data.get(\"id\"),\n \"url\": data.get(\"html_url\") or data.get(\"url\"),\n \"status\": data.get(\"state\") or \"open\",\n }\n # [/DEF:create_gitea_pull_request:Function]\n# [/DEF:GitServiceGiteaMixin:Class]\n# [/DEF:GitServiceGiteaMixin:Module]\n" }, { "contract_id": "_gitea_headers", "contract_type": "Function", "file_path": "backend/src/services/git/_gitea.py", - "start_line": 62, - "end_line": 76, + "start_line": 65, + "end_line": 79, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84432,8 +85441,8 @@ "contract_id": "_gitea_request", "contract_type": "Function", "file_path": "backend/src/services/git/_gitea.py", - "start_line": 78, - "end_line": 108, + "start_line": 81, + "end_line": 111, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84470,14 +85479,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_gitea_request:Function]\n # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping.\n # @PRE: method and endpoint are valid.\n # @POST: Returns decoded JSON payload.\n # @RETURN: Any\n async def _gitea_request(\n self, method: str, server_url: str, pat: str, endpoint: str,\n payload: Optional[Dict[str, Any]] = None,\n ) -> Any:\n base_url = self._normalize_git_server_url(server_url)\n url = f\"{base_url}/api/v1{endpoint}\"\n headers = self._gitea_headers(pat)\n try:\n async with httpx.AsyncClient(timeout=20.0) as client:\n response = await client.request(method=method, url=url, headers=headers, json=payload)\n except Exception as e:\n logger.error(f\"[gitea_request][Coherence:Failed] Network error: {e}\")\n raise HTTPException(status_code=503, detail=f\"Gitea API is unavailable: {str(e)}\")\n if response.status_code >= 400:\n detail = response.text\n try:\n parsed = response.json()\n detail = parsed.get(\"message\") or parsed.get(\"error\") or detail\n except Exception:\n pass\n logger.error(f\"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}\")\n raise HTTPException(status_code=response.status_code, detail=f\"Gitea API error: {detail}\")\n if response.status_code == 204:\n return None\n return response.json()\n # [/DEF:_gitea_request:Function]\n" + "body": " # [DEF:_gitea_request:Function]\n # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping.\n # @PRE: method and endpoint are valid.\n # @POST: Returns decoded JSON payload.\n # @RETURN: Any\n async def _gitea_request(\n self, method: str, server_url: str, pat: str, endpoint: str,\n payload: Optional[Dict[str, Any]] = None,\n ) -> Any:\n base_url = self._normalize_git_server_url(server_url)\n url = f\"{base_url}/api/v1{endpoint}\"\n headers = self._gitea_headers(pat)\n try:\n async with httpx.AsyncClient(timeout=20.0) as client:\n response = await client.request(method=method, url=url, headers=headers, json=payload)\n except Exception as e:\n log.explore(f\"Network error: {e}\", error=str(e))\n raise HTTPException(status_code=503, detail=f\"Gitea API is unavailable: {str(e)}\")\n if response.status_code >= 400:\n detail = response.text\n try:\n parsed = response.json()\n detail = parsed.get(\"message\") or parsed.get(\"error\") or detail\n except Exception:\n pass\n log.explore(f\"Gitea API error ({endpoint})\", payload={\"method\": method, \"status\": response.status_code}, error=detail)\n raise HTTPException(status_code=response.status_code, detail=f\"Gitea API error: {detail}\")\n if response.status_code == 204:\n return None\n return response.json()\n # [/DEF:_gitea_request:Function]\n" }, { "contract_id": "get_gitea_current_user", "contract_type": "Function", "file_path": "backend/src/services/git/_gitea.py", - "start_line": 110, - "end_line": 121, + "start_line": 113, + "end_line": 124, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84520,8 +85529,8 @@ "contract_id": "_gitea_branch_exists", "contract_type": "Function", "file_path": "backend/src/services/git/_gitea.py", - "start_line": 165, - "end_line": 181, + "start_line": 168, + "end_line": 184, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84564,8 +85573,8 @@ "contract_id": "_build_gitea_pr_404_detail", "contract_type": "Function", "file_path": "backend/src/services/git/_gitea.py", - "start_line": 183, - "end_line": 202, + "start_line": 186, + "end_line": 205, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -84608,7 +85617,7 @@ "contract_id": "create_gitea_pull_request", "contract_type": "Function", "file_path": "backend/src/services/git/_gitea.py", - "start_line": 204, + "start_line": 207, "end_line": 260, "tier": "TIER_1", "complexity": 1, @@ -84646,7 +85655,7 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:create_gitea_pull_request:Function]\n # @PURPOSE: Create pull request in Gitea.\n # @PRE: Config and remote URL are valid.\n # @POST: Returns normalized PR metadata.\n # @RETURN: Dict[str, Any]\n async def create_gitea_pull_request(\n self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,\n title: str, description: Optional[str] = None,\n ) -> Dict[str, Any]:\n identity = self._parse_remote_repo_identity(remote_url)\n req_payload = {\"title\": title, \"head\": from_branch, \"base\": to_branch, \"body\": description or \"\"}\n endpoint = f\"/repos/{identity['owner']}/{identity['repo']}/pulls\"\n active_server_url = server_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as exc:\n fallback_url = self._derive_server_url_from_remote(remote_url)\n normalized_primary = self._normalize_git_server_url(server_url)\n should_retry_with_fallback = (\n exc.status_code == 404 and fallback_url and fallback_url != normalized_primary\n )\n if should_retry_with_fallback:\n logger.warning(\n \"[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s\",\n fallback_url,\n )\n active_server_url = fallback_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as retry_exc:\n if retry_exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n else:\n if exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n if not isinstance(data, dict):\n raise HTTPException(status_code=500, detail=\"Unexpected Gitea response while creating pull request\")\n return {\n \"id\": data.get(\"number\") or data.get(\"id\"),\n \"url\": data.get(\"html_url\") or data.get(\"url\"),\n \"status\": data.get(\"state\") or \"open\",\n }\n # [/DEF:create_gitea_pull_request:Function]\n" + "body": " # [DEF:create_gitea_pull_request:Function]\n # @PURPOSE: Create pull request in Gitea.\n # @PRE: Config and remote URL are valid.\n # @POST: Returns normalized PR metadata.\n # @RETURN: Dict[str, Any]\n async def create_gitea_pull_request(\n self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str,\n title: str, description: Optional[str] = None,\n ) -> Dict[str, Any]:\n identity = self._parse_remote_repo_identity(remote_url)\n req_payload = {\"title\": title, \"head\": from_branch, \"base\": to_branch, \"body\": description or \"\"}\n endpoint = f\"/repos/{identity['owner']}/{identity['repo']}/pulls\"\n active_server_url = server_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as exc:\n fallback_url = self._derive_server_url_from_remote(remote_url)\n normalized_primary = self._normalize_git_server_url(server_url)\n should_retry_with_fallback = (\n exc.status_code == 404 and fallback_url and fallback_url != normalized_primary\n )\n if should_retry_with_fallback:\n log.explore(f\"Primary Gitea URL not found, retrying with remote host: {fallback_url}\", error=fallback_url)\n active_server_url = fallback_url\n try:\n data = await self._gitea_request(\"POST\", active_server_url, pat, endpoint, payload=req_payload)\n except HTTPException as retry_exc:\n if retry_exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n else:\n if exc.status_code == 404:\n branch_detail = await self._build_gitea_pr_404_detail(\n server_url=active_server_url, pat=pat,\n owner=identity[\"owner\"], repo=identity[\"repo\"],\n from_branch=from_branch, to_branch=to_branch,\n )\n if branch_detail:\n raise HTTPException(status_code=400, detail=branch_detail)\n raise\n if not isinstance(data, dict):\n raise HTTPException(status_code=500, detail=\"Unexpected Gitea response while creating pull request\")\n return {\n \"id\": data.get(\"number\") or data.get(\"id\"),\n \"url\": data.get(\"html_url\") or data.get(\"url\"),\n \"status\": data.get(\"state\") or \"open\",\n }\n # [/DEF:create_gitea_pull_request:Function]\n" }, { "contract_id": "GitServiceMergeMixin", @@ -84998,7 +86007,7 @@ "contract_type": "Module", "file_path": "backend/src/services/git/_status.py", "start_line": 1, - "end_line": 164, + "end_line": 167, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -85041,14 +86050,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitServiceStatusMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, status, diff, history, porcelain\n# @PURPOSE: Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues.\n# @RELATION: USED_BY -> [GitService]\n\nfrom typing import Any, Dict, List\nfrom datetime import datetime\nfrom git import Repo\nfrom src.core.logger import logger, belief_scope\n\n\n# [DEF:GitServiceStatusMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing repository status, diff, and commit history for GitService.\nclass GitServiceStatusMixin:\n # [DEF:_parse_status_porcelain:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists.\n # @PRE: `repo` is an open GitPython Repo instance.\n # @POST: Returns (staged, modified, untracked) tuple of file path lists.\n # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff(\"HEAD\") which internally\n # call git diff --cached, a flag unsupported in some Git environments (exit 129).\n # Using git status --porcelain is self-contained and avoids the --cached flag entirely.\n def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]:\n with belief_scope(\"GitService._parse_status_porcelain\"):\n staged: list[str] = []\n modified: list[str] = []\n untracked: list[str] = []\n try:\n output = repo.git.status(\"--porcelain\")\n except Exception:\n logger.warning(\"[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed\")\n return staged, modified, untracked\n for line in output.split(\"\\n\"):\n if not line:\n continue\n if line.startswith(\"??\"):\n untracked.append(line[2:].strip())\n continue\n if line.startswith(\"!!\"):\n continue\n if len(line) < 3:\n continue\n x = line[0]\n y = line[1]\n rest = line[3:]\n path = rest.split(\" -> \")[-1].strip() if \" -> \" in rest else rest.strip()\n if x != \" \" and x != \"?\":\n staged.append(path)\n if y != \" \" and y != \"?\":\n if path not in modified:\n modified.append(path)\n return staged, modified, untracked\n # [/DEF:_parse_status_porcelain:Function]\n\n # [DEF:get_status:Function]\n # @PURPOSE: Get current repository status (dirty files, untracked, etc.)\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a dictionary representing the Git status.\n # @RETURN: dict\n def get_status(self, dashboard_id: int) -> dict:\n with belief_scope(\"GitService.get_status\"):\n repo = self.get_repo(dashboard_id)\n has_commits = False\n try:\n repo.head.commit\n has_commits = True\n except (ValueError, Exception):\n has_commits = False\n current_branch = repo.active_branch.name\n tracking_branch = None\n has_upstream = False\n ahead_count = 0\n behind_count = 0\n try:\n tracking_branch = repo.active_branch.tracking_branch()\n has_upstream = tracking_branch is not None\n except Exception:\n tracking_branch = None\n has_upstream = False\n if has_upstream and tracking_branch is not None:\n try:\n ahead_count = sum(1 for _ in repo.iter_commits(f\"{tracking_branch.name}..{current_branch}\"))\n behind_count = sum(1 for _ in repo.iter_commits(f\"{current_branch}..{tracking_branch.name}\"))\n except Exception:\n ahead_count = 0\n behind_count = 0\n staged_files, modified_files, untracked_files = self._parse_status_porcelain(repo)\n is_dirty = bool(staged_files or modified_files or untracked_files)\n is_diverged = ahead_count > 0 and behind_count > 0\n if is_diverged:\n sync_state = \"DIVERGED\"\n elif behind_count > 0:\n sync_state = \"BEHIND_REMOTE\"\n elif ahead_count > 0:\n sync_state = \"AHEAD_REMOTE\"\n elif is_dirty or modified_files or staged_files or untracked_files:\n sync_state = \"CHANGES\"\n else:\n sync_state = \"SYNCED\"\n return {\n \"is_dirty\": is_dirty,\n \"untracked_files\": untracked_files,\n \"modified_files\": modified_files,\n \"staged_files\": staged_files,\n \"current_branch\": current_branch,\n \"upstream_branch\": tracking_branch.name if tracking_branch is not None else None,\n \"has_upstream\": has_upstream,\n \"ahead_count\": ahead_count,\n \"behind_count\": behind_count,\n \"is_diverged\": is_diverged,\n \"sync_state\": sync_state,\n }\n # [/DEF:get_status:Function]\n\n # [DEF:get_diff:Function]\n # @PURPOSE: Generate diff for a file or the whole repository.\n # @PARAM: file_path (str) - Optional specific file.\n # @PARAM: staged (bool) - Whether to show staged changes.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns the diff text as a string.\n # @RETURN: str\n def get_diff(self, dashboard_id: int, file_path: str = None, staged: bool = False) -> str:\n with belief_scope(\"GitService.get_diff\"):\n repo = self.get_repo(dashboard_id)\n diff_args = []\n if staged:\n diff_args.append(\"--staged\")\n if file_path:\n return repo.git.diff(*diff_args, \"--\", file_path)\n return repo.git.diff(*diff_args)\n # [/DEF:get_diff:Function]\n\n # [DEF:get_commit_history:Function]\n # @PURPOSE: Retrieve commit history for a repository.\n # @PARAM: limit (int) - Max number of commits to return.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of dictionaries for each commit in history.\n # @RETURN: List[dict]\n def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]:\n with belief_scope(\"GitService.get_commit_history\"):\n repo = self.get_repo(dashboard_id)\n commits = []\n try:\n if not repo.heads and not repo.remotes:\n return []\n for commit in repo.iter_commits(max_count=limit):\n commits.append({\n \"hash\": commit.hexsha,\n \"author\": commit.author.name,\n \"email\": commit.author.email,\n \"timestamp\": datetime.fromtimestamp(commit.committed_date),\n \"message\": commit.message.strip(),\n \"files_changed\": list(commit.stats.files.keys())\n })\n except Exception as e:\n logger.warning(f\"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}\")\n return []\n return commits\n # [/DEF:get_commit_history:Function]\n# [/DEF:GitServiceStatusMixin:Class]\n# [/DEF:GitServiceStatusMixin:Module]\n" + "body": "# [DEF:GitServiceStatusMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, status, diff, history, porcelain\n# @PURPOSE: Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues.\n# @RELATION: USED_BY -> [GitService]\n\nfrom typing import Any, Dict, List\nfrom datetime import datetime\nfrom git import Repo\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\n\nlog = MarkerLogger(\"GitStatus\")\n\n\n# [DEF:GitServiceStatusMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing repository status, diff, and commit history for GitService.\nclass GitServiceStatusMixin:\n # [DEF:_parse_status_porcelain:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists.\n # @PRE: `repo` is an open GitPython Repo instance.\n # @POST: Returns (staged, modified, untracked) tuple of file path lists.\n # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff(\"HEAD\") which internally\n # call git diff --cached, a flag unsupported in some Git environments (exit 129).\n # Using git status --porcelain is self-contained and avoids the --cached flag entirely.\n def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]:\n with belief_scope(\"GitService._parse_status_porcelain\"):\n staged: list[str] = []\n modified: list[str] = []\n untracked: list[str] = []\n try:\n output = repo.git.status(\"--porcelain\")\n except Exception:\n log.explore(\"git status --porcelain failed\", error=\"git status --porcelain command failed\")\n return staged, modified, untracked\n for line in output.split(\"\\n\"):\n if not line:\n continue\n if line.startswith(\"??\"):\n untracked.append(line[2:].strip())\n continue\n if line.startswith(\"!!\"):\n continue\n if len(line) < 3:\n continue\n x = line[0]\n y = line[1]\n rest = line[3:]\n path = rest.split(\" -> \")[-1].strip() if \" -> \" in rest else rest.strip()\n if x != \" \" and x != \"?\":\n staged.append(path)\n if y != \" \" and y != \"?\":\n if path not in modified:\n modified.append(path)\n return staged, modified, untracked\n # [/DEF:_parse_status_porcelain:Function]\n\n # [DEF:get_status:Function]\n # @PURPOSE: Get current repository status (dirty files, untracked, etc.)\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a dictionary representing the Git status.\n # @RETURN: dict\n def get_status(self, dashboard_id: int) -> dict:\n with belief_scope(\"GitService.get_status\"):\n repo = self.get_repo(dashboard_id)\n has_commits = False\n try:\n repo.head.commit\n has_commits = True\n except (ValueError, Exception):\n has_commits = False\n current_branch = repo.active_branch.name\n tracking_branch = None\n has_upstream = False\n ahead_count = 0\n behind_count = 0\n try:\n tracking_branch = repo.active_branch.tracking_branch()\n has_upstream = tracking_branch is not None\n except Exception:\n tracking_branch = None\n has_upstream = False\n if has_upstream and tracking_branch is not None:\n try:\n ahead_count = sum(1 for _ in repo.iter_commits(f\"{tracking_branch.name}..{current_branch}\"))\n behind_count = sum(1 for _ in repo.iter_commits(f\"{current_branch}..{tracking_branch.name}\"))\n except Exception:\n ahead_count = 0\n behind_count = 0\n staged_files, modified_files, untracked_files = self._parse_status_porcelain(repo)\n is_dirty = bool(staged_files or modified_files or untracked_files)\n is_diverged = ahead_count > 0 and behind_count > 0\n if is_diverged:\n sync_state = \"DIVERGED\"\n elif behind_count > 0:\n sync_state = \"BEHIND_REMOTE\"\n elif ahead_count > 0:\n sync_state = \"AHEAD_REMOTE\"\n elif is_dirty or modified_files or staged_files or untracked_files:\n sync_state = \"CHANGES\"\n else:\n sync_state = \"SYNCED\"\n return {\n \"is_dirty\": is_dirty,\n \"untracked_files\": untracked_files,\n \"modified_files\": modified_files,\n \"staged_files\": staged_files,\n \"current_branch\": current_branch,\n \"upstream_branch\": tracking_branch.name if tracking_branch is not None else None,\n \"has_upstream\": has_upstream,\n \"ahead_count\": ahead_count,\n \"behind_count\": behind_count,\n \"is_diverged\": is_diverged,\n \"sync_state\": sync_state,\n }\n # [/DEF:get_status:Function]\n\n # [DEF:get_diff:Function]\n # @PURPOSE: Generate diff for a file or the whole repository.\n # @PARAM: file_path (str) - Optional specific file.\n # @PARAM: staged (bool) - Whether to show staged changes.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns the diff text as a string.\n # @RETURN: str\n def get_diff(self, dashboard_id: int, file_path: str = None, staged: bool = False) -> str:\n with belief_scope(\"GitService.get_diff\"):\n repo = self.get_repo(dashboard_id)\n diff_args = []\n if staged:\n diff_args.append(\"--staged\")\n if file_path:\n return repo.git.diff(*diff_args, \"--\", file_path)\n return repo.git.diff(*diff_args)\n # [/DEF:get_diff:Function]\n\n # [DEF:get_commit_history:Function]\n # @PURPOSE: Retrieve commit history for a repository.\n # @PARAM: limit (int) - Max number of commits to return.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of dictionaries for each commit in history.\n # @RETURN: List[dict]\n def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]:\n with belief_scope(\"GitService.get_commit_history\"):\n repo = self.get_repo(dashboard_id)\n commits = []\n try:\n if not repo.heads and not repo.remotes:\n return []\n for commit in repo.iter_commits(max_count=limit):\n commits.append({\n \"hash\": commit.hexsha,\n \"author\": commit.author.name,\n \"email\": commit.author.email,\n \"timestamp\": datetime.fromtimestamp(commit.committed_date),\n \"message\": commit.message.strip(),\n \"files_changed\": list(commit.stats.files.keys())\n })\n except Exception as e:\n log.explore(f\"Could not retrieve commit history for dashboard {dashboard_id}: {e}\", error=str(e))\n return []\n return commits\n # [/DEF:get_commit_history:Function]\n# [/DEF:GitServiceStatusMixin:Class]\n# [/DEF:GitServiceStatusMixin:Module]\n" }, { "contract_id": "_parse_status_porcelain", "contract_type": "Function", "file_path": "backend/src/services/git/_status.py", - "start_line": 18, - "end_line": 56, + "start_line": 21, + "end_line": 59, "tier": "TIER_1", "complexity": 2, "metadata": { @@ -85080,14 +86089,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_parse_status_porcelain:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists.\n # @PRE: `repo` is an open GitPython Repo instance.\n # @POST: Returns (staged, modified, untracked) tuple of file path lists.\n # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff(\"HEAD\") which internally\n # call git diff --cached, a flag unsupported in some Git environments (exit 129).\n # Using git status --porcelain is self-contained and avoids the --cached flag entirely.\n def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]:\n with belief_scope(\"GitService._parse_status_porcelain\"):\n staged: list[str] = []\n modified: list[str] = []\n untracked: list[str] = []\n try:\n output = repo.git.status(\"--porcelain\")\n except Exception:\n logger.warning(\"[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed\")\n return staged, modified, untracked\n for line in output.split(\"\\n\"):\n if not line:\n continue\n if line.startswith(\"??\"):\n untracked.append(line[2:].strip())\n continue\n if line.startswith(\"!!\"):\n continue\n if len(line) < 3:\n continue\n x = line[0]\n y = line[1]\n rest = line[3:]\n path = rest.split(\" -> \")[-1].strip() if \" -> \" in rest else rest.strip()\n if x != \" \" and x != \"?\":\n staged.append(path)\n if y != \" \" and y != \"?\":\n if path not in modified:\n modified.append(path)\n return staged, modified, untracked\n # [/DEF:_parse_status_porcelain:Function]\n" + "body": " # [DEF:_parse_status_porcelain:Function]\n # @COMPLEXITY: 2\n # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists.\n # @PRE: `repo` is an open GitPython Repo instance.\n # @POST: Returns (staged, modified, untracked) tuple of file path lists.\n # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff(\"HEAD\") which internally\n # call git diff --cached, a flag unsupported in some Git environments (exit 129).\n # Using git status --porcelain is self-contained and avoids the --cached flag entirely.\n def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]:\n with belief_scope(\"GitService._parse_status_porcelain\"):\n staged: list[str] = []\n modified: list[str] = []\n untracked: list[str] = []\n try:\n output = repo.git.status(\"--porcelain\")\n except Exception:\n log.explore(\"git status --porcelain failed\", error=\"git status --porcelain command failed\")\n return staged, modified, untracked\n for line in output.split(\"\\n\"):\n if not line:\n continue\n if line.startswith(\"??\"):\n untracked.append(line[2:].strip())\n continue\n if line.startswith(\"!!\"):\n continue\n if len(line) < 3:\n continue\n x = line[0]\n y = line[1]\n rest = line[3:]\n path = rest.split(\" -> \")[-1].strip() if \" -> \" in rest else rest.strip()\n if x != \" \" and x != \"?\":\n staged.append(path)\n if y != \" \" and y != \"?\":\n if path not in modified:\n modified.append(path)\n return staged, modified, untracked\n # [/DEF:_parse_status_porcelain:Function]\n" }, { "contract_id": "get_status", "contract_type": "Function", "file_path": "backend/src/services/git/_status.py", - "start_line": 58, - "end_line": 116, + "start_line": 61, + "end_line": 119, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85130,8 +86139,8 @@ "contract_id": "get_diff", "contract_type": "Function", "file_path": "backend/src/services/git/_status.py", - "start_line": 118, - "end_line": 134, + "start_line": 121, + "end_line": 137, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85181,8 +86190,8 @@ "contract_id": "get_commit_history", "contract_type": "Function", "file_path": "backend/src/services/git/_status.py", - "start_line": 136, - "end_line": 162, + "start_line": 139, + "end_line": 165, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85226,14 +86235,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_commit_history:Function]\n # @PURPOSE: Retrieve commit history for a repository.\n # @PARAM: limit (int) - Max number of commits to return.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of dictionaries for each commit in history.\n # @RETURN: List[dict]\n def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]:\n with belief_scope(\"GitService.get_commit_history\"):\n repo = self.get_repo(dashboard_id)\n commits = []\n try:\n if not repo.heads and not repo.remotes:\n return []\n for commit in repo.iter_commits(max_count=limit):\n commits.append({\n \"hash\": commit.hexsha,\n \"author\": commit.author.name,\n \"email\": commit.author.email,\n \"timestamp\": datetime.fromtimestamp(commit.committed_date),\n \"message\": commit.message.strip(),\n \"files_changed\": list(commit.stats.files.keys())\n })\n except Exception as e:\n logger.warning(f\"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}\")\n return []\n return commits\n # [/DEF:get_commit_history:Function]\n" + "body": " # [DEF:get_commit_history:Function]\n # @PURPOSE: Retrieve commit history for a repository.\n # @PARAM: limit (int) - Max number of commits to return.\n # @PRE: Repository for dashboard_id exists.\n # @POST: Returns a list of dictionaries for each commit in history.\n # @RETURN: List[dict]\n def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]:\n with belief_scope(\"GitService.get_commit_history\"):\n repo = self.get_repo(dashboard_id)\n commits = []\n try:\n if not repo.heads and not repo.remotes:\n return []\n for commit in repo.iter_commits(max_count=limit):\n commits.append({\n \"hash\": commit.hexsha,\n \"author\": commit.author.name,\n \"email\": commit.author.email,\n \"timestamp\": datetime.fromtimestamp(commit.committed_date),\n \"message\": commit.message.strip(),\n \"files_changed\": list(commit.stats.files.keys())\n })\n except Exception as e:\n log.explore(f\"Could not retrieve commit history for dashboard {dashboard_id}: {e}\", error=str(e))\n return []\n return commits\n # [/DEF:get_commit_history:Function]\n" }, { "contract_id": "GitServiceSyncMixin", "contract_type": "Module", "file_path": "backend/src/services/git/_sync.py", "start_line": 1, - "end_line": 177, + "end_line": 194, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -85282,14 +86291,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitServiceSyncMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, push, pull, sync, remote\n# @PURPOSE: Push and pull operations for GitService with origin host auto-alignment.\n# @RELATION: USED_BY -> [GitService]\n# @RELATION: DEPENDS_ON -> [GitServiceUrlMixin]\n\nimport os\nfrom typing import Optional\nfrom git import Repo\nfrom git.exc import GitCommandError\nfrom fastapi import HTTPException\nfrom src.core.database import SessionLocal\nfrom src.core.logger import logger, belief_scope\nfrom src.models.git import GitRepository, GitServerConfig\n\n\n# [DEF:GitServiceSyncMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing push and pull operations with origin host alignment.\nclass GitServiceSyncMixin:\n # [DEF:push_changes:Function]\n # @PURPOSE: Push local commits to remote.\n # @PRE: Repository exists and has an 'origin' remote.\n # @POST: Local branch commits are pushed to origin.\n def push_changes(self, dashboard_id: int):\n with belief_scope(\"GitService.push_changes\"):\n repo = self.get_repo(dashboard_id)\n if not repo.heads:\n logger.warning(f\"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}\")\n return\n try:\n origin = repo.remote(name='origin')\n except ValueError:\n logger.error(f\"[push_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}\")\n raise HTTPException(status_code=400, detail=\"Remote 'origin' not configured\")\n try:\n origin_urls = list(origin.urls)\n except Exception:\n origin_urls = []\n binding_remote_url = None\n binding_config_id = None\n binding_config_url = None\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n binding_remote_url = db_repo.remote_url\n binding_config_id = db_repo.config_id\n db_config = (\n session.query(GitServerConfig)\n .filter(GitServerConfig.id == db_repo.config_id)\n .first()\n )\n if db_config:\n binding_config_url = db_config.url\n finally:\n session.close()\n except Exception as diag_error:\n logger.warning(\n \"[push_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s\",\n dashboard_id, diag_error,\n )\n realigned_origin_url = self._align_origin_host_with_config(\n dashboard_id=dashboard_id,\n origin=origin,\n config_url=binding_config_url,\n current_origin_url=(origin_urls[0] if origin_urls else None),\n binding_remote_url=binding_remote_url,\n )\n try:\n origin_urls = list(origin.urls)\n except Exception:\n origin_urls = []\n logger.info(\n \"[push_changes][Action] Push diagnostics dashboard=%s config_id=%s config_url=%s binding_remote_url=%s origin_urls=%s origin_realigned=%s\",\n dashboard_id, binding_config_id, binding_config_url, binding_remote_url, origin_urls, bool(realigned_origin_url),\n )\n try:\n current_branch = repo.active_branch\n logger.info(f\"[push_changes][Action] Pushing branch {current_branch.name} to origin\")\n tracking_branch = None\n try:\n tracking_branch = current_branch.tracking_branch()\n except Exception:\n tracking_branch = None\n if tracking_branch is None:\n repo.git.push(\"--set-upstream\", \"origin\", f\"{current_branch.name}:{current_branch.name}\")\n else:\n push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}')\n for info in push_info:\n if info.flags & info.ERROR:\n logger.error(f\"[push_changes][Coherence:Failed] Error pushing ref {info.remote_ref_string}: {info.summary}\")\n raise Exception(f\"Git push error for {info.remote_ref_string}: {info.summary}\")\n except GitCommandError as e:\n details = str(e)\n lowered = details.lower()\n if \"non-fast-forward\" in lowered or \"rejected\" in lowered:\n raise HTTPException(\n status_code=409,\n detail=\"Push rejected: remote branch contains newer commits. Run Pull first, resolve conflicts if any, then push again.\",\n )\n logger.error(f\"[push_changes][Coherence:Failed] Failed to push changes: {e}\")\n raise HTTPException(status_code=500, detail=f\"Git push failed: {details}\")\n except Exception as e:\n logger.error(f\"[push_changes][Coherence:Failed] Failed to push changes: {e}\")\n raise HTTPException(status_code=500, detail=f\"Git push failed: {str(e)}\")\n # [/DEF:push_changes:Function]\n\n # [DEF:pull_changes:Function]\n # @PURPOSE: Pull changes from remote.\n # @PRE: Repository exists and has an 'origin' remote.\n # @POST: Changes from origin are pulled and merged into the active branch.\n def pull_changes(self, dashboard_id: int):\n with belief_scope(\"GitService.pull_changes\"):\n repo = self.get_repo(dashboard_id)\n merge_head_path = os.path.join(repo.git_dir, \"MERGE_HEAD\")\n if os.path.exists(merge_head_path):\n payload = self._build_unfinished_merge_payload(repo)\n logger.warning(\n \"[pull_changes][Action] Unfinished merge detected for dashboard %s (repo_path=%s git_dir=%s branch=%s merge_head=%s merge_msg=%s)\",\n dashboard_id, payload[\"repository_path\"], payload[\"git_dir\"],\n payload[\"current_branch\"], payload[\"merge_head\"], payload[\"merge_message_preview\"],\n )\n raise HTTPException(status_code=409, detail=payload)\n try:\n origin = repo.remote(name='origin')\n current_branch = repo.active_branch.name\n try:\n origin_urls = list(origin.urls)\n except Exception:\n origin_urls = []\n logger.info(\n \"[pull_changes][Action] Pull diagnostics dashboard=%s repo_path=%s branch=%s origin_urls=%s\",\n dashboard_id, repo.working_tree_dir, current_branch, origin_urls,\n )\n origin.fetch(prune=True)\n remote_ref = f\"origin/{current_branch}\"\n has_remote_branch = any(ref.name == remote_ref for ref in repo.refs)\n logger.info(\n \"[pull_changes][Action] Pull remote branch check dashboard=%s branch=%s remote_ref=%s exists=%s\",\n dashboard_id, current_branch, remote_ref, has_remote_branch,\n )\n if not has_remote_branch:\n raise HTTPException(\n status_code=409,\n detail=f\"Remote branch '{current_branch}' does not exist yet. Push this branch first.\",\n )\n logger.info(f\"[pull_changes][Action] Pulling changes from origin/{current_branch}\")\n repo.git.pull(\"--no-rebase\", \"origin\", current_branch)\n except ValueError:\n logger.error(f\"[pull_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}\")\n raise HTTPException(status_code=400, detail=\"Remote 'origin' not configured\")\n except GitCommandError as e:\n details = str(e)\n lowered = details.lower()\n if \"conflict\" in lowered or \"not possible to fast-forward\" in lowered:\n raise HTTPException(\n status_code=409,\n detail=\"Pull requires conflict resolution. Resolve conflicts in repository and repeat operation.\",\n )\n logger.error(f\"[pull_changes][Coherence:Failed] Failed to pull changes: {e}\")\n raise HTTPException(status_code=500, detail=f\"Git pull failed: {details}\")\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"[pull_changes][Coherence:Failed] Failed to pull changes: {e}\")\n raise HTTPException(status_code=500, detail=f\"Git pull failed: {str(e)}\")\n # [/DEF:pull_changes:Function]\n# [/DEF:GitServiceSyncMixin:Class]\n# [/DEF:GitServiceSyncMixin:Module]\n" + "body": "# [DEF:GitServiceSyncMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, push, pull, sync, remote\n# @PURPOSE: Push and pull operations for GitService with origin host auto-alignment.\n# @RELATION: USED_BY -> [GitService]\n# @RELATION: DEPENDS_ON -> [GitServiceUrlMixin]\n\nimport os\nfrom typing import Optional\nfrom git import Repo\nfrom git.exc import GitCommandError\nfrom fastapi import HTTPException\nfrom src.core.database import SessionLocal\nfrom src.core.logger import belief_scope\nfrom src.core.cot_logger import MarkerLogger\n\nlog = MarkerLogger(\"GitSync\")\nfrom src.models.git import GitRepository, GitServerConfig\n\n\n# [DEF:GitServiceSyncMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Mixin providing push and pull operations with origin host alignment.\nclass GitServiceSyncMixin:\n # [DEF:push_changes:Function]\n # @PURPOSE: Push local commits to remote.\n # @PRE: Repository exists and has an 'origin' remote.\n # @POST: Local branch commits are pushed to origin.\n def push_changes(self, dashboard_id: int):\n with belief_scope(\"GitService.push_changes\"):\n repo = self.get_repo(dashboard_id)\n if not repo.heads:\n log.explore(\"No local branches to push\", payload={\"dashboard_id\": dashboard_id}, error=\"Repository has no local branch heads to push\")\n return\n try:\n origin = repo.remote(name='origin')\n except ValueError:\n log.explore(\"Remote 'origin' not found\", payload={\"dashboard_id\": dashboard_id}, error=\"Git remote 'origin' is not configured for this repository\")\n raise HTTPException(status_code=400, detail=\"Remote 'origin' not configured\")\n try:\n origin_urls = list(origin.urls)\n except Exception:\n origin_urls = []\n binding_remote_url = None\n binding_config_id = None\n binding_config_url = None\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n binding_remote_url = db_repo.remote_url\n binding_config_id = db_repo.config_id\n db_config = (\n session.query(GitServerConfig)\n .filter(GitServerConfig.id == db_repo.config_id)\n .first()\n )\n if db_config:\n binding_config_url = db_config.url\n finally:\n session.close()\n except Exception as diag_error:\n log.explore(\"Failed to load repository binding diagnostics\", payload={\"dashboard_id\": dashboard_id}, error=str(diag_error))\n realigned_origin_url = self._align_origin_host_with_config(\n dashboard_id=dashboard_id,\n origin=origin,\n config_url=binding_config_url,\n current_origin_url=(origin_urls[0] if origin_urls else None),\n binding_remote_url=binding_remote_url,\n )\n try:\n origin_urls = list(origin.urls)\n except Exception:\n origin_urls = []\n log.reason(\n \"Push diagnostics\",\n payload={\n \"dashboard_id\": dashboard_id, \"config_id\": binding_config_id,\n \"config_url\": binding_config_url, \"binding_remote_url\": binding_remote_url,\n \"origin_urls\": origin_urls, \"origin_realigned\": bool(realigned_origin_url),\n },\n )\n try:\n current_branch = repo.active_branch\n log.reason(\"Pushing branch\", payload={\"branch\": current_branch.name, \"origin\": True})\n tracking_branch = None\n try:\n tracking_branch = current_branch.tracking_branch()\n except Exception:\n tracking_branch = None\n if tracking_branch is None:\n repo.git.push(\"--set-upstream\", \"origin\", f\"{current_branch.name}:{current_branch.name}\")\n else:\n push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}')\n for info in push_info:\n if info.flags & info.ERROR:\n log.explore(\"Error pushing ref\", payload={\"ref\": info.remote_ref_string, \"summary\": info.summary}, error=f\"Git push error: {info.summary}\")\n raise Exception(f\"Git push error for {info.remote_ref_string}: {info.summary}\")\n except GitCommandError as e:\n details = str(e)\n lowered = details.lower()\n if \"non-fast-forward\" in lowered or \"rejected\" in lowered:\n raise HTTPException(\n status_code=409,\n detail=\"Push rejected: remote branch contains newer commits. Run Pull first, resolve conflicts if any, then push again.\",\n )\n log.explore(\"Failed to push changes\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Git push failed: {details}\")\n except Exception as e:\n log.explore(\"Failed to push changes\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Git push failed: {str(e)}\")\n # [/DEF:push_changes:Function]\n\n # [DEF:pull_changes:Function]\n # @PURPOSE: Pull changes from remote.\n # @PRE: Repository exists and has an 'origin' remote.\n # @POST: Changes from origin are pulled and merged into the active branch.\n def pull_changes(self, dashboard_id: int):\n with belief_scope(\"GitService.pull_changes\"):\n repo = self.get_repo(dashboard_id)\n merge_head_path = os.path.join(repo.git_dir, \"MERGE_HEAD\")\n if os.path.exists(merge_head_path):\n payload = self._build_unfinished_merge_payload(repo)\n log.explore(\"Unfinished merge detected\", error=\"Unfinished merge state found\",\n payload={\n \"dashboard_id\": dashboard_id,\n \"repo_path\": payload[\"repository_path\"],\n \"git_dir\": payload[\"git_dir\"],\n \"branch\": payload[\"current_branch\"],\n \"merge_head\": payload[\"merge_head\"],\n })\n raise HTTPException(status_code=409, detail=payload)\n try:\n origin = repo.remote(name='origin')\n current_branch = repo.active_branch.name\n try:\n origin_urls = list(origin.urls)\n except Exception:\n origin_urls = []\n log.reason(\n \"Pull diagnostics\",\n payload={\n \"dashboard_id\": dashboard_id,\n \"repo_path\": repo.working_tree_dir,\n \"branch\": current_branch,\n \"origin_urls\": origin_urls,\n },\n )\n origin.fetch(prune=True)\n remote_ref = f\"origin/{current_branch}\"\n has_remote_branch = any(ref.name == remote_ref for ref in repo.refs)\n log.reason(\n \"Pull remote branch check\",\n payload={\n \"dashboard_id\": dashboard_id,\n \"branch\": current_branch,\n \"remote_ref\": remote_ref,\n \"exists\": has_remote_branch,\n },\n )\n if not has_remote_branch:\n raise HTTPException(\n status_code=409,\n detail=f\"Remote branch '{current_branch}' does not exist yet. Push this branch first.\",\n )\n log.reason(\"Pulling changes\", payload={\"branch\": current_branch})\n repo.git.pull(\"--no-rebase\", \"origin\", current_branch)\n except ValueError:\n log.explore(\"Remote 'origin' not found\", payload={\"dashboard_id\": dashboard_id}, error=\"Git remote 'origin' is not configured for this repository\")\n raise HTTPException(status_code=400, detail=\"Remote 'origin' not configured\")\n except GitCommandError as e:\n details = str(e)\n lowered = details.lower()\n if \"conflict\" in lowered or \"not possible to fast-forward\" in lowered:\n raise HTTPException(\n status_code=409,\n detail=\"Pull requires conflict resolution. Resolve conflicts in repository and repeat operation.\",\n )\n log.explore(\"Failed to pull changes\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Git pull failed: {details}\")\n except HTTPException:\n raise\n except Exception as e:\n log.explore(\"Failed to pull changes\", error=str(e))\n raise HTTPException(status_code=500, detail=f\"Git pull failed: {str(e)}\")\n # [/DEF:pull_changes:Function]\n# [/DEF:GitServiceSyncMixin:Class]\n# [/DEF:GitServiceSyncMixin:Module]\n" }, { "contract_id": "GitServiceUrlMixin", "contract_type": "Module", "file_path": "backend/src/services/git/_url.py", "start_line": 1, - "end_line": 212, + "end_line": 206, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -85378,14 +86387,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:GitServiceUrlMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, url, parsing, normalization, host_alignment\n# @PURPOSE: URL helper mixin for GitService — parse, normalize, align, and strip credentials from Git URLs.\n# @RELATION: USED_BY -> [GitServiceSyncMixin]\n# @RELATION: USED_BY -> [GitServiceGiteaMixin]\n# @RELATION: USED_BY -> [GitServiceRemoteMixin]\n\nimport os\nfrom typing import Any, Dict, List, Optional\nfrom urllib.parse import quote, urlparse\nfrom fastapi import HTTPException\nfrom src.core.database import SessionLocal\nfrom src.core.logger import logger, belief_scope\nfrom src.models.git import GitRepository, GitServerConfig\n\n# [DEF:GitServiceUrlMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL.\nclass GitServiceUrlMixin:\n # [DEF:_extract_http_host:Function]\n # @PURPOSE: Extract normalized host[:port] from HTTP(S) URL.\n # @PRE: url_value may be empty.\n # @POST: Returns lowercase host token or None.\n # @RETURN: Optional[str]\n def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]:\n normalized = str(url_value or \"\").strip()\n if not normalized:\n return None\n try:\n parsed = urlparse(normalized)\n except Exception:\n return None\n if parsed.scheme not in {\"http\", \"https\"}:\n return None\n host = parsed.hostname\n if not host:\n return None\n if parsed.port:\n return f\"{host.lower()}:{parsed.port}\"\n return host.lower()\n # [/DEF:_extract_http_host:Function]\n\n # [DEF:_strip_url_credentials:Function]\n # @PURPOSE: Remove credentials from URL while preserving scheme/host/path.\n # @PRE: url_value may contain credentials.\n # @POST: Returns URL without username/password.\n # @RETURN: str\n def _strip_url_credentials(self, url_value: str) -> str:\n normalized = str(url_value or \"\").strip()\n if not normalized:\n return normalized\n try:\n parsed = urlparse(normalized)\n except Exception:\n return normalized\n if parsed.scheme not in {\"http\", \"https\"} or not parsed.hostname:\n return normalized\n host = parsed.hostname\n if parsed.port:\n host = f\"{host}:{parsed.port}\"\n return parsed._replace(netloc=host).geturl()\n # [/DEF:_strip_url_credentials:Function]\n\n # [DEF:_replace_host_in_url:Function]\n # @PURPOSE: Replace source URL host with host from configured server URL.\n # @PRE: source_url and config_url are HTTP(S) URLs.\n # @POST: Returns source URL with updated host (credentials preserved) or None.\n # @RETURN: Optional[str]\n def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]:\n source = str(source_url or \"\").strip()\n config = str(config_url or \"\").strip()\n if not source or not config:\n return None\n try:\n source_parsed = urlparse(source)\n config_parsed = urlparse(config)\n except Exception:\n return None\n if source_parsed.scheme not in {\"http\", \"https\"} or config_parsed.scheme not in {\"http\", \"https\"}:\n return None\n if not source_parsed.hostname or not config_parsed.hostname:\n return None\n target_host = config_parsed.hostname\n if config_parsed.port:\n target_host = f\"{target_host}:{config_parsed.port}\"\n auth_part = \"\"\n if source_parsed.username:\n auth_part = quote(source_parsed.username, safe=\"\")\n if source_parsed.password is not None:\n auth_part = f\"{auth_part}:{quote(source_parsed.password, safe='')}\"\n auth_part = f\"{auth_part}@\"\n new_netloc = f\"{auth_part}{target_host}\"\n return source_parsed._replace(netloc=new_netloc).geturl()\n # [/DEF:_replace_host_in_url:Function]\n\n # [DEF:_align_origin_host_with_config:Function]\n # @PURPOSE: Auto-align local origin host to configured Git server host when they drift.\n # @PRE: origin remote exists.\n # @POST: origin URL host updated and DB binding normalized when mismatch detected.\n # @RETURN: Optional[str]\n def _align_origin_host_with_config(\n self,\n dashboard_id: int,\n origin,\n config_url: Optional[str],\n current_origin_url: Optional[str],\n binding_remote_url: Optional[str],\n ) -> Optional[str]:\n config_host = self._extract_http_host(config_url)\n source_origin_url = str(current_origin_url or \"\").strip() or str(binding_remote_url or \"\").strip()\n origin_host = self._extract_http_host(source_origin_url)\n if not config_host or not origin_host:\n return None\n if config_host == origin_host:\n return None\n aligned_url = self._replace_host_in_url(source_origin_url, config_url)\n if not aligned_url:\n return None\n logger.warning(\n \"[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url\",\n dashboard_id, config_host, origin_host,\n )\n try:\n origin.set_url(aligned_url)\n except Exception as e:\n logger.warning(\n \"[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s\",\n dashboard_id, e,\n )\n return None\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.remote_url = self._strip_url_credentials(aligned_url)\n session.commit()\n finally:\n session.close()\n except Exception as e:\n logger.warning(\n \"[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s\",\n dashboard_id, e,\n )\n return aligned_url\n # [/DEF:_align_origin_host_with_config:Function]\n\n # [DEF:_parse_remote_repo_identity:Function]\n # @PURPOSE: Parse owner/repo from remote URL for Git server API operations.\n # @PRE: remote_url is a valid git URL.\n # @POST: Returns owner/repo tokens.\n # @RETURN: Dict[str, str]\n def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]:\n normalized = str(remote_url or \"\").strip()\n if not normalized:\n raise HTTPException(status_code=400, detail=\"Repository remote_url is empty\")\n if normalized.startswith(\"git@\"):\n path = normalized.split(\":\", 1)[1] if \":\" in normalized else \"\"\n else:\n parsed = urlparse(normalized)\n path = parsed.path or \"\"\n path = path.strip(\"/\")\n if path.endswith(\".git\"):\n path = path[:-4]\n parts = [segment for segment in path.split(\"/\") if segment]\n if len(parts) < 2:\n raise HTTPException(status_code=400, detail=f\"Cannot parse repository owner/name from remote URL: {remote_url}\")\n owner = parts[0]\n repo = parts[-1]\n namespace = \"/\".join(parts[:-1])\n return {\"owner\": owner, \"repo\": repo, \"namespace\": namespace, \"full_name\": f\"{namespace}/{repo}\"}\n # [/DEF:_parse_remote_repo_identity:Function]\n\n # [DEF:_derive_server_url_from_remote:Function]\n # @PURPOSE: Build API base URL from remote repository URL without credentials.\n # @PRE: remote_url may be any git URL.\n # @POST: Returns normalized http(s) base URL or None when derivation is impossible.\n # @RETURN: Optional[str]\n def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]:\n normalized = str(remote_url or \"\").strip()\n if not normalized or normalized.startswith(\"git@\"):\n return None\n parsed = urlparse(normalized)\n if parsed.scheme not in {\"http\", \"https\"}:\n return None\n if not parsed.hostname:\n return None\n netloc = parsed.hostname\n if parsed.port:\n netloc = f\"{netloc}:{parsed.port}\"\n return f\"{parsed.scheme}://{netloc}\".rstrip(\"/\")\n # [/DEF:_derive_server_url_from_remote:Function]\n\n # [DEF:_normalize_git_server_url:Function]\n # @PURPOSE: Normalize Git server URL for provider API calls.\n # @PRE: raw_url is non-empty.\n # @POST: Returns URL without trailing slash.\n # @RETURN: str\n def _normalize_git_server_url(self, raw_url: str) -> str:\n normalized = (raw_url or \"\").strip()\n if not normalized:\n raise HTTPException(status_code=400, detail=\"Git server URL is required\")\n return normalized.rstrip(\"/\")\n # [/DEF:_normalize_git_server_url:Function]\n# [/DEF:GitServiceUrlMixin:Class]\n# [/DEF:GitServiceUrlMixin:Module]\n" + "body": "# [DEF:GitServiceUrlMixin:Module]\n# @COMPLEXITY: 3\n# @LAYER: Infra\n# @SEMANTICS: git, url, parsing, normalization, host_alignment\n# @PURPOSE: URL helper mixin for GitService — parse, normalize, align, and strip credentials from Git URLs.\n# @RELATION: USED_BY -> [GitServiceSyncMixin]\n# @RELATION: USED_BY -> [GitServiceGiteaMixin]\n# @RELATION: USED_BY -> [GitServiceRemoteMixin]\n\nimport os\nfrom typing import Any, Dict, List, Optional\nfrom urllib.parse import quote, urlparse\nfrom fastapi import HTTPException\nfrom src.core.database import SessionLocal\nfrom src.core.cot_logger import MarkerLogger\nfrom src.core.logger import belief_scope\n\nlog = MarkerLogger(\"GitUrl\")\nfrom src.models.git import GitRepository, GitServerConfig\n\n# [DEF:GitServiceUrlMixin:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL.\nclass GitServiceUrlMixin:\n # [DEF:_extract_http_host:Function]\n # @PURPOSE: Extract normalized host[:port] from HTTP(S) URL.\n # @PRE: url_value may be empty.\n # @POST: Returns lowercase host token or None.\n # @RETURN: Optional[str]\n def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]:\n normalized = str(url_value or \"\").strip()\n if not normalized:\n return None\n try:\n parsed = urlparse(normalized)\n except Exception:\n return None\n if parsed.scheme not in {\"http\", \"https\"}:\n return None\n host = parsed.hostname\n if not host:\n return None\n if parsed.port:\n return f\"{host.lower()}:{parsed.port}\"\n return host.lower()\n # [/DEF:_extract_http_host:Function]\n\n # [DEF:_strip_url_credentials:Function]\n # @PURPOSE: Remove credentials from URL while preserving scheme/host/path.\n # @PRE: url_value may contain credentials.\n # @POST: Returns URL without username/password.\n # @RETURN: str\n def _strip_url_credentials(self, url_value: str) -> str:\n normalized = str(url_value or \"\").strip()\n if not normalized:\n return normalized\n try:\n parsed = urlparse(normalized)\n except Exception:\n return normalized\n if parsed.scheme not in {\"http\", \"https\"} or not parsed.hostname:\n return normalized\n host = parsed.hostname\n if parsed.port:\n host = f\"{host}:{parsed.port}\"\n return parsed._replace(netloc=host).geturl()\n # [/DEF:_strip_url_credentials:Function]\n\n # [DEF:_replace_host_in_url:Function]\n # @PURPOSE: Replace source URL host with host from configured server URL.\n # @PRE: source_url and config_url are HTTP(S) URLs.\n # @POST: Returns source URL with updated host (credentials preserved) or None.\n # @RETURN: Optional[str]\n def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]:\n source = str(source_url or \"\").strip()\n config = str(config_url or \"\").strip()\n if not source or not config:\n return None\n try:\n source_parsed = urlparse(source)\n config_parsed = urlparse(config)\n except Exception:\n return None\n if source_parsed.scheme not in {\"http\", \"https\"} or config_parsed.scheme not in {\"http\", \"https\"}:\n return None\n if not source_parsed.hostname or not config_parsed.hostname:\n return None\n target_host = config_parsed.hostname\n if config_parsed.port:\n target_host = f\"{target_host}:{config_parsed.port}\"\n auth_part = \"\"\n if source_parsed.username:\n auth_part = quote(source_parsed.username, safe=\"\")\n if source_parsed.password is not None:\n auth_part = f\"{auth_part}:{quote(source_parsed.password, safe='')}\"\n auth_part = f\"{auth_part}@\"\n new_netloc = f\"{auth_part}{target_host}\"\n return source_parsed._replace(netloc=new_netloc).geturl()\n # [/DEF:_replace_host_in_url:Function]\n\n # [DEF:_align_origin_host_with_config:Function]\n # @PURPOSE: Auto-align local origin host to configured Git server host when they drift.\n # @PRE: origin remote exists.\n # @POST: origin URL host updated and DB binding normalized when mismatch detected.\n # @RETURN: Optional[str]\n def _align_origin_host_with_config(\n self,\n dashboard_id: int,\n origin,\n config_url: Optional[str],\n current_origin_url: Optional[str],\n binding_remote_url: Optional[str],\n ) -> Optional[str]:\n config_host = self._extract_http_host(config_url)\n source_origin_url = str(current_origin_url or \"\").strip() or str(binding_remote_url or \"\").strip()\n origin_host = self._extract_http_host(source_origin_url)\n if not config_host or not origin_host:\n return None\n if config_host == origin_host:\n return None\n aligned_url = self._replace_host_in_url(source_origin_url, config_url)\n if not aligned_url:\n return None\n log.explore(f\"Host mismatch for dashboard {dashboard_id}: config_host={config_host} origin_host={origin_host}, applying origin.set_url\", error=f\"Host mismatch for dashboard {dashboard_id}\")\n try:\n origin.set_url(aligned_url)\n except Exception as e:\n log.explore(f\"Failed to set origin URL for dashboard {dashboard_id}: {e}\", error=str(e))\n return None\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.remote_url = self._strip_url_credentials(aligned_url)\n session.commit()\n finally:\n session.close()\n except Exception as e:\n log.explore(f\"Failed to persist aligned remote_url for dashboard {dashboard_id}: {e}\", error=str(e))\n return aligned_url\n # [/DEF:_align_origin_host_with_config:Function]\n\n # [DEF:_parse_remote_repo_identity:Function]\n # @PURPOSE: Parse owner/repo from remote URL for Git server API operations.\n # @PRE: remote_url is a valid git URL.\n # @POST: Returns owner/repo tokens.\n # @RETURN: Dict[str, str]\n def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]:\n normalized = str(remote_url or \"\").strip()\n if not normalized:\n raise HTTPException(status_code=400, detail=\"Repository remote_url is empty\")\n if normalized.startswith(\"git@\"):\n path = normalized.split(\":\", 1)[1] if \":\" in normalized else \"\"\n else:\n parsed = urlparse(normalized)\n path = parsed.path or \"\"\n path = path.strip(\"/\")\n if path.endswith(\".git\"):\n path = path[:-4]\n parts = [segment for segment in path.split(\"/\") if segment]\n if len(parts) < 2:\n raise HTTPException(status_code=400, detail=f\"Cannot parse repository owner/name from remote URL: {remote_url}\")\n owner = parts[0]\n repo = parts[-1]\n namespace = \"/\".join(parts[:-1])\n return {\"owner\": owner, \"repo\": repo, \"namespace\": namespace, \"full_name\": f\"{namespace}/{repo}\"}\n # [/DEF:_parse_remote_repo_identity:Function]\n\n # [DEF:_derive_server_url_from_remote:Function]\n # @PURPOSE: Build API base URL from remote repository URL without credentials.\n # @PRE: remote_url may be any git URL.\n # @POST: Returns normalized http(s) base URL or None when derivation is impossible.\n # @RETURN: Optional[str]\n def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]:\n normalized = str(remote_url or \"\").strip()\n if not normalized or normalized.startswith(\"git@\"):\n return None\n parsed = urlparse(normalized)\n if parsed.scheme not in {\"http\", \"https\"}:\n return None\n if not parsed.hostname:\n return None\n netloc = parsed.hostname\n if parsed.port:\n netloc = f\"{netloc}:{parsed.port}\"\n return f\"{parsed.scheme}://{netloc}\".rstrip(\"/\")\n # [/DEF:_derive_server_url_from_remote:Function]\n\n # [DEF:_normalize_git_server_url:Function]\n # @PURPOSE: Normalize Git server URL for provider API calls.\n # @PRE: raw_url is non-empty.\n # @POST: Returns URL without trailing slash.\n # @RETURN: str\n def _normalize_git_server_url(self, raw_url: str) -> str:\n normalized = (raw_url or \"\").strip()\n if not normalized:\n raise HTTPException(status_code=400, detail=\"Git server URL is required\")\n return normalized.rstrip(\"/\")\n # [/DEF:_normalize_git_server_url:Function]\n# [/DEF:GitServiceUrlMixin:Class]\n# [/DEF:GitServiceUrlMixin:Module]\n" }, { "contract_id": "_extract_http_host", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 22, - "end_line": 43, + "start_line": 25, + "end_line": 46, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85428,8 +86437,8 @@ "contract_id": "_strip_url_credentials", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 45, - "end_line": 64, + "start_line": 48, + "end_line": 67, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85472,8 +86481,8 @@ "contract_id": "_replace_host_in_url", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 66, - "end_line": 96, + "start_line": 69, + "end_line": 99, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85516,8 +86525,8 @@ "contract_id": "_align_origin_host_with_config", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 98, - "end_line": 152, + "start_line": 101, + "end_line": 146, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85554,14 +86563,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_align_origin_host_with_config:Function]\n # @PURPOSE: Auto-align local origin host to configured Git server host when they drift.\n # @PRE: origin remote exists.\n # @POST: origin URL host updated and DB binding normalized when mismatch detected.\n # @RETURN: Optional[str]\n def _align_origin_host_with_config(\n self,\n dashboard_id: int,\n origin,\n config_url: Optional[str],\n current_origin_url: Optional[str],\n binding_remote_url: Optional[str],\n ) -> Optional[str]:\n config_host = self._extract_http_host(config_url)\n source_origin_url = str(current_origin_url or \"\").strip() or str(binding_remote_url or \"\").strip()\n origin_host = self._extract_http_host(source_origin_url)\n if not config_host or not origin_host:\n return None\n if config_host == origin_host:\n return None\n aligned_url = self._replace_host_in_url(source_origin_url, config_url)\n if not aligned_url:\n return None\n logger.warning(\n \"[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url\",\n dashboard_id, config_host, origin_host,\n )\n try:\n origin.set_url(aligned_url)\n except Exception as e:\n logger.warning(\n \"[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s\",\n dashboard_id, e,\n )\n return None\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.remote_url = self._strip_url_credentials(aligned_url)\n session.commit()\n finally:\n session.close()\n except Exception as e:\n logger.warning(\n \"[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s\",\n dashboard_id, e,\n )\n return aligned_url\n # [/DEF:_align_origin_host_with_config:Function]\n" + "body": " # [DEF:_align_origin_host_with_config:Function]\n # @PURPOSE: Auto-align local origin host to configured Git server host when they drift.\n # @PRE: origin remote exists.\n # @POST: origin URL host updated and DB binding normalized when mismatch detected.\n # @RETURN: Optional[str]\n def _align_origin_host_with_config(\n self,\n dashboard_id: int,\n origin,\n config_url: Optional[str],\n current_origin_url: Optional[str],\n binding_remote_url: Optional[str],\n ) -> Optional[str]:\n config_host = self._extract_http_host(config_url)\n source_origin_url = str(current_origin_url or \"\").strip() or str(binding_remote_url or \"\").strip()\n origin_host = self._extract_http_host(source_origin_url)\n if not config_host or not origin_host:\n return None\n if config_host == origin_host:\n return None\n aligned_url = self._replace_host_in_url(source_origin_url, config_url)\n if not aligned_url:\n return None\n log.explore(f\"Host mismatch for dashboard {dashboard_id}: config_host={config_host} origin_host={origin_host}, applying origin.set_url\", error=f\"Host mismatch for dashboard {dashboard_id}\")\n try:\n origin.set_url(aligned_url)\n except Exception as e:\n log.explore(f\"Failed to set origin URL for dashboard {dashboard_id}: {e}\", error=str(e))\n return None\n try:\n session = SessionLocal()\n try:\n db_repo = (\n session.query(GitRepository)\n .filter(GitRepository.dashboard_id == int(dashboard_id))\n .first()\n )\n if db_repo:\n db_repo.remote_url = self._strip_url_credentials(aligned_url)\n session.commit()\n finally:\n session.close()\n except Exception as e:\n log.explore(f\"Failed to persist aligned remote_url for dashboard {dashboard_id}: {e}\", error=str(e))\n return aligned_url\n # [/DEF:_align_origin_host_with_config:Function]\n" }, { "contract_id": "_parse_remote_repo_identity", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 154, - "end_line": 178, + "start_line": 148, + "end_line": 172, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85604,8 +86613,8 @@ "contract_id": "_derive_server_url_from_remote", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 180, - "end_line": 198, + "start_line": 174, + "end_line": 192, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -85648,8 +86657,8 @@ "contract_id": "_normalize_git_server_url", "contract_type": "Function", "file_path": "backend/src/services/git/_url.py", - "start_line": 200, - "end_line": 210, + "start_line": 194, + "end_line": 204, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -92584,7 +93593,7 @@ "contract_type": "Module", "file_path": "backend/src/services/resource_service.py", "start_line": 1, - "end_line": 498, + "end_line": 495, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -92653,14 +93662,14 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:ResourceServiceModule:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: service, resources, dashboards, datasets, tasks, git\n# @PURPOSE: Shared service for fetching resource data with Git status and task status\n# @LAYER: Service\n# @RELATION: DEPENDS_ON ->[SupersetClient]\n# @RELATION: DEPENDS_ON ->[TaskManagerPackage]\n# @RELATION: DEPENDS_ON ->[TaskManagerModels]\n# @RELATION: DEPENDS_ON ->[GitService]\n# @INVARIANT: All resources include metadata about their current state\n\n# [SECTION: IMPORTS]\nfrom typing import List, Dict, Optional, Any\nfrom datetime import datetime, timezone\nfrom ..core.superset_client import SupersetClient\nfrom ..core.task_manager.models import Task\nfrom ..services.git_service import GitService\nfrom ..core.logger import logger, belief_scope\n# [/SECTION]\n\n# [DEF:ResourceService:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Provides centralized access to resource data with enhanced metadata\n# @RELATION: DEPENDS_ON ->[SupersetClient]\n# @RELATION: DEPENDS_ON ->[GitService]\nclass ResourceService:\n \n # [DEF:ResourceService_init:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initialize the resource service with dependencies\n # @PRE: None\n # @POST: ResourceService is ready to fetch resources\n def __init__(self):\n with belief_scope(\"ResourceService.__init__\"):\n self.git_service = GitService()\n logger.info(\"[ResourceService][Action] Initialized ResourceService\")\n # [/DEF:ResourceService_init:Function]\n \n # [DEF:get_dashboards_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch dashboards from environment with Git status and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of dashboards with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Dashboards with git_status and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummary]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_dashboards_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary(require_slug=require_slug)\n \n # Enhance each dashboard with Git status and task status\n result = []\n for dashboard in dashboards:\n # dashboard is already a dict, no need to call .dict()\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get('id')\n \n # Git status can be skipped for list endpoints and loaded lazily on UI side.\n if include_git_status:\n git_status = self._get_git_status_for_dashboard(dashboard_id)\n dashboard_dict['git_status'] = git_status\n else:\n dashboard_dict['git_status'] = None\n \n # Show status of the latest LLM validation for this dashboard.\n last_task = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n dashboard_dict['last_task'] = last_task\n \n result.append(dashboard_dict)\n \n logger.info(f\"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status\")\n return result\n # [/DEF:get_dashboards_with_status:Function]\n\n # [DEF:get_dashboards_page_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.\n # @PRE: env is valid; page >= 1; page_size > 0.\n # @POST: Returns page items plus total counters without scanning all pages locally.\n # @PARAM: env (Environment) - Source environment.\n # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status.\n # @PARAM: page (int) - 1-based page number.\n # @PARAM: page_size (int) - Page size.\n # @RETURN: Dict[str, Any] - {\"dashboards\": List[Dict], \"total\": int, \"total_pages\": int}\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummaryPage]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_page_with_status(\n self,\n env: Any,\n tasks: Optional[List[Task]] = None,\n page: int = 1,\n page_size: int = 10,\n search: Optional[str] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\n \"get_dashboards_page_with_status\",\n f\"env={env.id}, page={page}, page_size={page_size}, search={search}\",\n ):\n client = SupersetClient(env)\n total, dashboards_page = client.get_dashboards_summary_page(\n page=page,\n page_size=page_size,\n search=search,\n require_slug=require_slug,\n )\n\n result = []\n for dashboard in dashboards_page:\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get(\"id\")\n\n if include_git_status:\n dashboard_dict[\"git_status\"] = self._get_git_status_for_dashboard(dashboard_id)\n else:\n dashboard_dict[\"git_status\"] = None\n\n dashboard_dict[\"last_task\"] = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n result.append(dashboard_dict)\n\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n logger.info(\n \"[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)\",\n page,\n total_pages,\n len(result),\n total,\n )\n return {\n \"dashboards\": result,\n \"total\": total,\n \"total_pages\": total_pages,\n }\n # [/DEF:get_dashboards_page_with_status:Function]\n\n # [DEF:_get_last_llm_task_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment\n # @PRE: dashboard_id is a valid integer identifier\n # @POST: Returns the newest llm_dashboard_validation task summary or None\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @PARAM: env_id (Optional[str]) - Environment ID to match task params\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n # @RELATION: CALLS ->[_normalize_validation_status]\n # @RELATION: CALLS ->[_normalize_task_status]\n def _get_last_llm_task_for_dashboard(\n self,\n dashboard_id: int,\n env_id: Optional[str],\n tasks: Optional[List[Task]] = None,\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n\n dashboard_id_str = str(dashboard_id)\n matched_tasks = []\n\n for task in tasks:\n if getattr(task, \"plugin_id\", None) != \"llm_dashboard_validation\":\n continue\n\n params = getattr(task, \"params\", {}) or {}\n if str(params.get(\"dashboard_id\")) != dashboard_id_str:\n continue\n\n if env_id is not None:\n task_env = params.get(\"environment_id\") or params.get(\"env\")\n if str(task_env) != str(env_id):\n continue\n\n matched_tasks.append(task)\n\n if not matched_tasks:\n return None\n\n def _task_time(task_obj: Any) -> datetime:\n raw_time = (\n getattr(task_obj, \"started_at\", None)\n or getattr(task_obj, \"finished_at\", None)\n or getattr(task_obj, \"created_at\", None)\n )\n return self._normalize_datetime_for_compare(raw_time)\n\n projected_tasks = []\n for task in matched_tasks:\n raw_result = getattr(task, \"result\", None)\n validation_status = None\n if isinstance(raw_result, dict):\n validation_status = self._normalize_validation_status(raw_result.get(\"status\"))\n projected_tasks.append(\n (\n task,\n validation_status,\n _task_time(task),\n )\n )\n\n projected_tasks.sort(key=lambda item: item[2], reverse=True)\n latest_task, latest_validation_status, _ = projected_tasks[0]\n decisive_task = next(\n (\n item for item in projected_tasks\n if item[1] in {\"PASS\", \"WARN\", \"FAIL\"}\n ),\n None,\n )\n validation_status = latest_validation_status\n if validation_status == \"UNKNOWN\" and decisive_task is not None:\n validation_status = decisive_task[1]\n\n return {\n \"task_id\": str(getattr(latest_task, \"id\", \"\")),\n \"status\": self._normalize_task_status(getattr(latest_task, \"status\", \"\")),\n \"validation_status\": validation_status,\n }\n # [/DEF:_get_last_llm_task_for_dashboard:Function]\n\n # [DEF:_normalize_task_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections\n # @PRE: raw_status can be enum or string\n # @POST: Returns uppercase status without enum class prefix\n # @PARAM: raw_status (Any) - Raw task status object/value\n # @RETURN: str - Normalized status token\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_task_status(self, raw_status: Any) -> str:\n if raw_status is None:\n return \"\"\n value = getattr(raw_status, \"value\", raw_status)\n status_text = str(value).strip()\n if \".\" in status_text:\n status_text = status_text.split(\".\")[-1]\n return status_text.upper()\n # [/DEF:_normalize_task_status:Function]\n\n # [DEF:_normalize_validation_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN\n # @PRE: raw_status can be any scalar type\n # @POST: Returns normalized validation status token or None\n # @PARAM: raw_status (Any) - Raw validation status from task result\n # @RETURN: Optional[str] - PASS|FAIL|WARN|UNKNOWN\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_validation_status(self, raw_status: Any) -> Optional[str]:\n if raw_status is None:\n return None\n status_text = str(raw_status).strip().upper()\n if status_text in {\"PASS\", \"FAIL\", \"WARN\"}:\n return status_text\n return \"UNKNOWN\"\n # [/DEF:_normalize_validation_status:Function]\n\n # [DEF:_normalize_datetime_for_compare:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.\n # @PRE: value may be datetime or any scalar.\n # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.\n # @PARAM: value (Any) - Candidate datetime-like value.\n # @RETURN: datetime - UTC-aware comparable datetime.\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n # @RELATION: USED_BY ->[_get_last_task_for_resource]\n def _normalize_datetime_for_compare(self, value: Any) -> datetime:\n if isinstance(value, datetime):\n if value.tzinfo is None:\n return value.replace(tzinfo=timezone.utc)\n return value.astimezone(timezone.utc)\n return datetime.min.replace(tzinfo=timezone.utc)\n # [/DEF:_normalize_datetime_for_compare:Function]\n \n # [DEF:get_datasets_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch datasets from environment with mapping progress and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of datasets with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary]\n # @RELATION: CALLS ->[_get_last_task_for_resource]\n async def get_datasets_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_datasets_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n datasets = client.get_datasets_summary()\n \n # Enhance each dataset with task status\n result = []\n for dataset in datasets:\n # dataset is already a dict, no need to call .dict()\n dataset_dict = dataset\n dataset_id = dataset_dict.get('id')\n \n # Get last task status\n last_task = self._get_last_task_for_resource(\n f\"dataset-{dataset_id}\", \n tasks\n )\n dataset_dict['last_task'] = last_task\n \n result.append(dataset_dict)\n \n logger.info(f\"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status\")\n return result\n # [/DEF:get_datasets_with_status:Function]\n \n # [DEF:get_activity_summary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get summary of active and recent tasks for the activity indicator\n # @PRE: tasks is a list of Task objects\n # @POST: Returns summary with active_count and recent_tasks\n # @PARAM: tasks (List[Task]) - List of tasks to summarize\n # @RETURN: Dict - Activity summary\n # @RELATION: CALLS ->[_extract_resource_name_from_task]\n # @RELATION: CALLS ->[_extract_resource_type_from_task]\n def get_activity_summary(self, tasks: List[Task]) -> Dict[str, Any]:\n with belief_scope(\"get_activity_summary\"):\n # Count active (RUNNING, WAITING_INPUT) tasks\n active_tasks = [\n t for t in tasks \n if t.status in ['RUNNING', 'WAITING_INPUT']\n ]\n \n # Get recent tasks (last 5)\n recent_tasks = sorted(\n tasks, \n key=lambda t: t.created_at, \n reverse=True\n )[:5]\n \n # Format recent tasks for frontend\n recent_tasks_formatted = []\n for task in recent_tasks:\n resource_name = self._extract_resource_name_from_task(task)\n recent_tasks_formatted.append({\n 'task_id': str(task.id),\n 'resource_name': resource_name,\n 'resource_type': self._extract_resource_type_from_task(task),\n 'status': task.status,\n 'started_at': task.created_at.isoformat() if task.created_at else None\n })\n \n return {\n 'active_count': len(active_tasks),\n 'recent_tasks': recent_tasks_formatted\n }\n # [/DEF:get_activity_summary:Function]\n \n # [DEF:_get_git_status_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get Git sync status for a dashboard\n # @PRE: dashboard_id is a valid integer\n # @POST: Returns git status or None if no repo exists\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @RETURN: Optional[Dict] - Git status with branch and sync_status\n # @RELATION: CALLS ->[get_repo]\n def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:\n try:\n repo = self.git_service.get_repo(dashboard_id)\n if not repo:\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n \n # Check if there are uncommitted changes\n try:\n # Get current branch\n branch = repo.active_branch.name\n \n # Check for uncommitted changes\n is_dirty = repo.is_dirty()\n has_changes_for_commit = repo.is_dirty(untracked_files=True)\n \n # Check for unpushed commits\n unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0\n \n if is_dirty or unpushed > 0:\n sync_status = 'DIFF'\n else:\n sync_status = 'OK'\n \n return {\n 'branch': branch,\n 'sync_status': sync_status,\n 'has_repo': True,\n 'has_changes_for_commit': has_changes_for_commit\n }\n except Exception:\n logger.warning(f\"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}\")\n return {\n 'branch': None,\n 'sync_status': 'ERROR',\n 'has_repo': True,\n 'has_changes_for_commit': False\n }\n except Exception:\n # No repo exists for this dashboard\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n # [/DEF:_get_git_status_for_dashboard:Function]\n \n # [DEF:_get_last_task_for_resource:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get the most recent task for a specific resource\n # @PRE: resource_id is a valid string\n # @POST: Returns task summary or None if no tasks found\n # @PARAM: resource_id (str) - The resource identifier (e.g., \"dashboard-123\")\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n def _get_last_task_for_resource(\n self,\n resource_id: str,\n tasks: Optional[List[Task]] = None\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n \n # Filter tasks for this resource\n resource_tasks = []\n for task in tasks:\n params = task.params or {}\n if params.get('resource_id') == resource_id:\n resource_tasks.append(task)\n \n if not resource_tasks:\n return None\n \n # Get most recent task with timezone-safe comparison.\n last_task = max(\n resource_tasks,\n key=lambda t: self._normalize_datetime_for_compare(getattr(t, \"created_at\", None)),\n )\n \n return {\n 'task_id': str(last_task.id),\n 'status': last_task.status\n }\n # [/DEF:_get_last_task_for_resource:Function]\n \n # [DEF:_extract_resource_name_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource name from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource name or task ID\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource name or fallback\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_name_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_name', f\"Task {task.id}\")\n # [/DEF:_extract_resource_name_from_task:Function]\n \n # [DEF:_extract_resource_type_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource type from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource type or 'unknown'\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource type\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_type_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_type', 'unknown')\n # [/DEF:_extract_resource_type_from_task:Function]\n# [/DEF:ResourceService:Class]\n# [/DEF:ResourceServiceModule:Module]\n" + "body": "# [DEF:ResourceServiceModule:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: service, resources, dashboards, datasets, tasks, git\n# @PURPOSE: Shared service for fetching resource data with Git status and task status\n# @LAYER: Service\n# @RELATION: DEPENDS_ON ->[SupersetClient]\n# @RELATION: DEPENDS_ON ->[TaskManagerPackage]\n# @RELATION: DEPENDS_ON ->[TaskManagerModels]\n# @RELATION: DEPENDS_ON ->[GitService]\n# @INVARIANT: All resources include metadata about their current state\n\n# [SECTION: IMPORTS]\nfrom typing import List, Dict, Optional, Any\nfrom datetime import datetime, timezone\nfrom ..core.superset_client import SupersetClient\nfrom ..core.task_manager.models import Task\nfrom ..services.git_service import GitService\nfrom ..core.cot_logger import MarkerLogger\nfrom ..core.logger import logger, belief_scope\n\nlog = MarkerLogger(\"ResourceService\")\n# [/SECTION]\n\n# [DEF:ResourceService:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Provides centralized access to resource data with enhanced metadata\n# @RELATION: DEPENDS_ON ->[SupersetClient]\n# @RELATION: DEPENDS_ON ->[GitService]\nclass ResourceService:\n \n # [DEF:ResourceService_init:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initialize the resource service with dependencies\n # @PRE: None\n # @POST: ResourceService is ready to fetch resources\n def __init__(self):\n with belief_scope(\"ResourceService.__init__\"):\n self.git_service = GitService()\n log.reason(\"Initialized ResourceService\")\n # [/DEF:ResourceService_init:Function]\n \n # [DEF:get_dashboards_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch dashboards from environment with Git status and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of dashboards with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Dashboards with git_status and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummary]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_dashboards_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary(require_slug=require_slug)\n \n # Enhance each dashboard with Git status and task status\n result = []\n for dashboard in dashboards:\n # dashboard is already a dict, no need to call .dict()\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get('id')\n \n # Git status can be skipped for list endpoints and loaded lazily on UI side.\n if include_git_status:\n git_status = self._get_git_status_for_dashboard(dashboard_id)\n dashboard_dict['git_status'] = git_status\n else:\n dashboard_dict['git_status'] = None\n \n # Show status of the latest LLM validation for this dashboard.\n last_task = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n dashboard_dict['last_task'] = last_task\n \n result.append(dashboard_dict)\n \n log.reflect(f\"Fetched {len(result)} dashboards with status\")\n return result\n # [/DEF:get_dashboards_with_status:Function]\n\n # [DEF:get_dashboards_page_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.\n # @PRE: env is valid; page >= 1; page_size > 0.\n # @POST: Returns page items plus total counters without scanning all pages locally.\n # @PARAM: env (Environment) - Source environment.\n # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status.\n # @PARAM: page (int) - 1-based page number.\n # @PARAM: page_size (int) - Page size.\n # @RETURN: Dict[str, Any] - {\"dashboards\": List[Dict], \"total\": int, \"total_pages\": int}\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummaryPage]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_page_with_status(\n self,\n env: Any,\n tasks: Optional[List[Task]] = None,\n page: int = 1,\n page_size: int = 10,\n search: Optional[str] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\n \"get_dashboards_page_with_status\",\n f\"env={env.id}, page={page}, page_size={page_size}, search={search}\",\n ):\n client = SupersetClient(env)\n total, dashboards_page = client.get_dashboards_summary_page(\n page=page,\n page_size=page_size,\n search=search,\n require_slug=require_slug,\n )\n\n result = []\n for dashboard in dashboards_page:\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get(\"id\")\n\n if include_git_status:\n dashboard_dict[\"git_status\"] = self._get_git_status_for_dashboard(dashboard_id)\n else:\n dashboard_dict[\"git_status\"] = None\n\n dashboard_dict[\"last_task\"] = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n result.append(dashboard_dict)\n\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n log.reflect(f\"Fetched dashboards page {page}/{total_pages} ({len(result)} items, total={total})\")\n return {\n \"dashboards\": result,\n \"total\": total,\n \"total_pages\": total_pages,\n }\n # [/DEF:get_dashboards_page_with_status:Function]\n\n # [DEF:_get_last_llm_task_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment\n # @PRE: dashboard_id is a valid integer identifier\n # @POST: Returns the newest llm_dashboard_validation task summary or None\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @PARAM: env_id (Optional[str]) - Environment ID to match task params\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n # @RELATION: CALLS ->[_normalize_validation_status]\n # @RELATION: CALLS ->[_normalize_task_status]\n def _get_last_llm_task_for_dashboard(\n self,\n dashboard_id: int,\n env_id: Optional[str],\n tasks: Optional[List[Task]] = None,\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n\n dashboard_id_str = str(dashboard_id)\n matched_tasks = []\n\n for task in tasks:\n if getattr(task, \"plugin_id\", None) != \"llm_dashboard_validation\":\n continue\n\n params = getattr(task, \"params\", {}) or {}\n if str(params.get(\"dashboard_id\")) != dashboard_id_str:\n continue\n\n if env_id is not None:\n task_env = params.get(\"environment_id\") or params.get(\"env\")\n if str(task_env) != str(env_id):\n continue\n\n matched_tasks.append(task)\n\n if not matched_tasks:\n return None\n\n def _task_time(task_obj: Any) -> datetime:\n raw_time = (\n getattr(task_obj, \"started_at\", None)\n or getattr(task_obj, \"finished_at\", None)\n or getattr(task_obj, \"created_at\", None)\n )\n return self._normalize_datetime_for_compare(raw_time)\n\n projected_tasks = []\n for task in matched_tasks:\n raw_result = getattr(task, \"result\", None)\n validation_status = None\n if isinstance(raw_result, dict):\n validation_status = self._normalize_validation_status(raw_result.get(\"status\"))\n projected_tasks.append(\n (\n task,\n validation_status,\n _task_time(task),\n )\n )\n\n projected_tasks.sort(key=lambda item: item[2], reverse=True)\n latest_task, latest_validation_status, _ = projected_tasks[0]\n decisive_task = next(\n (\n item for item in projected_tasks\n if item[1] in {\"PASS\", \"WARN\", \"FAIL\"}\n ),\n None,\n )\n validation_status = latest_validation_status\n if validation_status == \"UNKNOWN\" and decisive_task is not None:\n validation_status = decisive_task[1]\n\n return {\n \"task_id\": str(getattr(latest_task, \"id\", \"\")),\n \"status\": self._normalize_task_status(getattr(latest_task, \"status\", \"\")),\n \"validation_status\": validation_status,\n }\n # [/DEF:_get_last_llm_task_for_dashboard:Function]\n\n # [DEF:_normalize_task_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections\n # @PRE: raw_status can be enum or string\n # @POST: Returns uppercase status without enum class prefix\n # @PARAM: raw_status (Any) - Raw task status object/value\n # @RETURN: str - Normalized status token\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_task_status(self, raw_status: Any) -> str:\n if raw_status is None:\n return \"\"\n value = getattr(raw_status, \"value\", raw_status)\n status_text = str(value).strip()\n if \".\" in status_text:\n status_text = status_text.split(\".\")[-1]\n return status_text.upper()\n # [/DEF:_normalize_task_status:Function]\n\n # [DEF:_normalize_validation_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN\n # @PRE: raw_status can be any scalar type\n # @POST: Returns normalized validation status token or None\n # @PARAM: raw_status (Any) - Raw validation status from task result\n # @RETURN: Optional[str] - PASS|FAIL|WARN|UNKNOWN\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_validation_status(self, raw_status: Any) -> Optional[str]:\n if raw_status is None:\n return None\n status_text = str(raw_status).strip().upper()\n if status_text in {\"PASS\", \"FAIL\", \"WARN\"}:\n return status_text\n return \"UNKNOWN\"\n # [/DEF:_normalize_validation_status:Function]\n\n # [DEF:_normalize_datetime_for_compare:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.\n # @PRE: value may be datetime or any scalar.\n # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.\n # @PARAM: value (Any) - Candidate datetime-like value.\n # @RETURN: datetime - UTC-aware comparable datetime.\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n # @RELATION: USED_BY ->[_get_last_task_for_resource]\n def _normalize_datetime_for_compare(self, value: Any) -> datetime:\n if isinstance(value, datetime):\n if value.tzinfo is None:\n return value.replace(tzinfo=timezone.utc)\n return value.astimezone(timezone.utc)\n return datetime.min.replace(tzinfo=timezone.utc)\n # [/DEF:_normalize_datetime_for_compare:Function]\n \n # [DEF:get_datasets_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch datasets from environment with mapping progress and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of datasets with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary]\n # @RELATION: CALLS ->[_get_last_task_for_resource]\n async def get_datasets_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_datasets_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n datasets = client.get_datasets_summary()\n \n # Enhance each dataset with task status\n result = []\n for dataset in datasets:\n # dataset is already a dict, no need to call .dict()\n dataset_dict = dataset\n dataset_id = dataset_dict.get('id')\n \n # Get last task status\n last_task = self._get_last_task_for_resource(\n f\"dataset-{dataset_id}\", \n tasks\n )\n dataset_dict['last_task'] = last_task\n \n result.append(dataset_dict)\n \n log.reflect(f\"Fetched {len(result)} datasets with status\")\n return result\n # [/DEF:get_datasets_with_status:Function]\n \n # [DEF:get_activity_summary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get summary of active and recent tasks for the activity indicator\n # @PRE: tasks is a list of Task objects\n # @POST: Returns summary with active_count and recent_tasks\n # @PARAM: tasks (List[Task]) - List of tasks to summarize\n # @RETURN: Dict - Activity summary\n # @RELATION: CALLS ->[_extract_resource_name_from_task]\n # @RELATION: CALLS ->[_extract_resource_type_from_task]\n def get_activity_summary(self, tasks: List[Task]) -> Dict[str, Any]:\n with belief_scope(\"get_activity_summary\"):\n # Count active (RUNNING, WAITING_INPUT) tasks\n active_tasks = [\n t for t in tasks \n if t.status in ['RUNNING', 'WAITING_INPUT']\n ]\n \n # Get recent tasks (last 5)\n recent_tasks = sorted(\n tasks, \n key=lambda t: t.created_at, \n reverse=True\n )[:5]\n \n # Format recent tasks for frontend\n recent_tasks_formatted = []\n for task in recent_tasks:\n resource_name = self._extract_resource_name_from_task(task)\n recent_tasks_formatted.append({\n 'task_id': str(task.id),\n 'resource_name': resource_name,\n 'resource_type': self._extract_resource_type_from_task(task),\n 'status': task.status,\n 'started_at': task.created_at.isoformat() if task.created_at else None\n })\n \n return {\n 'active_count': len(active_tasks),\n 'recent_tasks': recent_tasks_formatted\n }\n # [/DEF:get_activity_summary:Function]\n \n # [DEF:_get_git_status_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get Git sync status for a dashboard\n # @PRE: dashboard_id is a valid integer\n # @POST: Returns git status or None if no repo exists\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @RETURN: Optional[Dict] - Git status with branch and sync_status\n # @RELATION: CALLS ->[get_repo]\n def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:\n try:\n repo = self.git_service.get_repo(dashboard_id)\n if not repo:\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n \n # Check if there are uncommitted changes\n try:\n # Get current branch\n branch = repo.active_branch.name\n \n # Check for uncommitted changes\n is_dirty = repo.is_dirty()\n has_changes_for_commit = repo.is_dirty(untracked_files=True)\n \n # Check for unpushed commits\n unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0\n \n if is_dirty or unpushed > 0:\n sync_status = 'DIFF'\n else:\n sync_status = 'OK'\n \n return {\n 'branch': branch,\n 'sync_status': sync_status,\n 'has_repo': True,\n 'has_changes_for_commit': has_changes_for_commit\n }\n except Exception:\n log.explore(f\"Failed to get git status for dashboard {dashboard_id}\", error=\"Git status check failed\")\n return {\n 'branch': None,\n 'sync_status': 'ERROR',\n 'has_repo': True,\n 'has_changes_for_commit': False\n }\n except Exception:\n # No repo exists for this dashboard\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n # [/DEF:_get_git_status_for_dashboard:Function]\n \n # [DEF:_get_last_task_for_resource:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get the most recent task for a specific resource\n # @PRE: resource_id is a valid string\n # @POST: Returns task summary or None if no tasks found\n # @PARAM: resource_id (str) - The resource identifier (e.g., \"dashboard-123\")\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n def _get_last_task_for_resource(\n self,\n resource_id: str,\n tasks: Optional[List[Task]] = None\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n \n # Filter tasks for this resource\n resource_tasks = []\n for task in tasks:\n params = task.params or {}\n if params.get('resource_id') == resource_id:\n resource_tasks.append(task)\n \n if not resource_tasks:\n return None\n \n # Get most recent task with timezone-safe comparison.\n last_task = max(\n resource_tasks,\n key=lambda t: self._normalize_datetime_for_compare(getattr(t, \"created_at\", None)),\n )\n \n return {\n 'task_id': str(last_task.id),\n 'status': last_task.status\n }\n # [/DEF:_get_last_task_for_resource:Function]\n \n # [DEF:_extract_resource_name_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource name from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource name or task ID\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource name or fallback\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_name_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_name', f\"Task {task.id}\")\n # [/DEF:_extract_resource_name_from_task:Function]\n \n # [DEF:_extract_resource_type_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource type from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource type or 'unknown'\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource type\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_type_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_type', 'unknown')\n # [/DEF:_extract_resource_type_from_task:Function]\n# [/DEF:ResourceService:Class]\n# [/DEF:ResourceServiceModule:Module]\n" }, { "contract_id": "ResourceService", "contract_type": "Class", "file_path": "backend/src/services/resource_service.py", - "start_line": 21, - "end_line": 497, + "start_line": 24, + "end_line": 494, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -92683,14 +93692,14 @@ ], "schema_warnings": [], "anchor_syntax": "def", - "body": "# [DEF:ResourceService:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Provides centralized access to resource data with enhanced metadata\n# @RELATION: DEPENDS_ON ->[SupersetClient]\n# @RELATION: DEPENDS_ON ->[GitService]\nclass ResourceService:\n \n # [DEF:ResourceService_init:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initialize the resource service with dependencies\n # @PRE: None\n # @POST: ResourceService is ready to fetch resources\n def __init__(self):\n with belief_scope(\"ResourceService.__init__\"):\n self.git_service = GitService()\n logger.info(\"[ResourceService][Action] Initialized ResourceService\")\n # [/DEF:ResourceService_init:Function]\n \n # [DEF:get_dashboards_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch dashboards from environment with Git status and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of dashboards with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Dashboards with git_status and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummary]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_dashboards_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary(require_slug=require_slug)\n \n # Enhance each dashboard with Git status and task status\n result = []\n for dashboard in dashboards:\n # dashboard is already a dict, no need to call .dict()\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get('id')\n \n # Git status can be skipped for list endpoints and loaded lazily on UI side.\n if include_git_status:\n git_status = self._get_git_status_for_dashboard(dashboard_id)\n dashboard_dict['git_status'] = git_status\n else:\n dashboard_dict['git_status'] = None\n \n # Show status of the latest LLM validation for this dashboard.\n last_task = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n dashboard_dict['last_task'] = last_task\n \n result.append(dashboard_dict)\n \n logger.info(f\"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status\")\n return result\n # [/DEF:get_dashboards_with_status:Function]\n\n # [DEF:get_dashboards_page_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.\n # @PRE: env is valid; page >= 1; page_size > 0.\n # @POST: Returns page items plus total counters without scanning all pages locally.\n # @PARAM: env (Environment) - Source environment.\n # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status.\n # @PARAM: page (int) - 1-based page number.\n # @PARAM: page_size (int) - Page size.\n # @RETURN: Dict[str, Any] - {\"dashboards\": List[Dict], \"total\": int, \"total_pages\": int}\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummaryPage]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_page_with_status(\n self,\n env: Any,\n tasks: Optional[List[Task]] = None,\n page: int = 1,\n page_size: int = 10,\n search: Optional[str] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\n \"get_dashboards_page_with_status\",\n f\"env={env.id}, page={page}, page_size={page_size}, search={search}\",\n ):\n client = SupersetClient(env)\n total, dashboards_page = client.get_dashboards_summary_page(\n page=page,\n page_size=page_size,\n search=search,\n require_slug=require_slug,\n )\n\n result = []\n for dashboard in dashboards_page:\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get(\"id\")\n\n if include_git_status:\n dashboard_dict[\"git_status\"] = self._get_git_status_for_dashboard(dashboard_id)\n else:\n dashboard_dict[\"git_status\"] = None\n\n dashboard_dict[\"last_task\"] = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n result.append(dashboard_dict)\n\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n logger.info(\n \"[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)\",\n page,\n total_pages,\n len(result),\n total,\n )\n return {\n \"dashboards\": result,\n \"total\": total,\n \"total_pages\": total_pages,\n }\n # [/DEF:get_dashboards_page_with_status:Function]\n\n # [DEF:_get_last_llm_task_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment\n # @PRE: dashboard_id is a valid integer identifier\n # @POST: Returns the newest llm_dashboard_validation task summary or None\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @PARAM: env_id (Optional[str]) - Environment ID to match task params\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n # @RELATION: CALLS ->[_normalize_validation_status]\n # @RELATION: CALLS ->[_normalize_task_status]\n def _get_last_llm_task_for_dashboard(\n self,\n dashboard_id: int,\n env_id: Optional[str],\n tasks: Optional[List[Task]] = None,\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n\n dashboard_id_str = str(dashboard_id)\n matched_tasks = []\n\n for task in tasks:\n if getattr(task, \"plugin_id\", None) != \"llm_dashboard_validation\":\n continue\n\n params = getattr(task, \"params\", {}) or {}\n if str(params.get(\"dashboard_id\")) != dashboard_id_str:\n continue\n\n if env_id is not None:\n task_env = params.get(\"environment_id\") or params.get(\"env\")\n if str(task_env) != str(env_id):\n continue\n\n matched_tasks.append(task)\n\n if not matched_tasks:\n return None\n\n def _task_time(task_obj: Any) -> datetime:\n raw_time = (\n getattr(task_obj, \"started_at\", None)\n or getattr(task_obj, \"finished_at\", None)\n or getattr(task_obj, \"created_at\", None)\n )\n return self._normalize_datetime_for_compare(raw_time)\n\n projected_tasks = []\n for task in matched_tasks:\n raw_result = getattr(task, \"result\", None)\n validation_status = None\n if isinstance(raw_result, dict):\n validation_status = self._normalize_validation_status(raw_result.get(\"status\"))\n projected_tasks.append(\n (\n task,\n validation_status,\n _task_time(task),\n )\n )\n\n projected_tasks.sort(key=lambda item: item[2], reverse=True)\n latest_task, latest_validation_status, _ = projected_tasks[0]\n decisive_task = next(\n (\n item for item in projected_tasks\n if item[1] in {\"PASS\", \"WARN\", \"FAIL\"}\n ),\n None,\n )\n validation_status = latest_validation_status\n if validation_status == \"UNKNOWN\" and decisive_task is not None:\n validation_status = decisive_task[1]\n\n return {\n \"task_id\": str(getattr(latest_task, \"id\", \"\")),\n \"status\": self._normalize_task_status(getattr(latest_task, \"status\", \"\")),\n \"validation_status\": validation_status,\n }\n # [/DEF:_get_last_llm_task_for_dashboard:Function]\n\n # [DEF:_normalize_task_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections\n # @PRE: raw_status can be enum or string\n # @POST: Returns uppercase status without enum class prefix\n # @PARAM: raw_status (Any) - Raw task status object/value\n # @RETURN: str - Normalized status token\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_task_status(self, raw_status: Any) -> str:\n if raw_status is None:\n return \"\"\n value = getattr(raw_status, \"value\", raw_status)\n status_text = str(value).strip()\n if \".\" in status_text:\n status_text = status_text.split(\".\")[-1]\n return status_text.upper()\n # [/DEF:_normalize_task_status:Function]\n\n # [DEF:_normalize_validation_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN\n # @PRE: raw_status can be any scalar type\n # @POST: Returns normalized validation status token or None\n # @PARAM: raw_status (Any) - Raw validation status from task result\n # @RETURN: Optional[str] - PASS|FAIL|WARN|UNKNOWN\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_validation_status(self, raw_status: Any) -> Optional[str]:\n if raw_status is None:\n return None\n status_text = str(raw_status).strip().upper()\n if status_text in {\"PASS\", \"FAIL\", \"WARN\"}:\n return status_text\n return \"UNKNOWN\"\n # [/DEF:_normalize_validation_status:Function]\n\n # [DEF:_normalize_datetime_for_compare:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.\n # @PRE: value may be datetime or any scalar.\n # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.\n # @PARAM: value (Any) - Candidate datetime-like value.\n # @RETURN: datetime - UTC-aware comparable datetime.\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n # @RELATION: USED_BY ->[_get_last_task_for_resource]\n def _normalize_datetime_for_compare(self, value: Any) -> datetime:\n if isinstance(value, datetime):\n if value.tzinfo is None:\n return value.replace(tzinfo=timezone.utc)\n return value.astimezone(timezone.utc)\n return datetime.min.replace(tzinfo=timezone.utc)\n # [/DEF:_normalize_datetime_for_compare:Function]\n \n # [DEF:get_datasets_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch datasets from environment with mapping progress and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of datasets with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary]\n # @RELATION: CALLS ->[_get_last_task_for_resource]\n async def get_datasets_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_datasets_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n datasets = client.get_datasets_summary()\n \n # Enhance each dataset with task status\n result = []\n for dataset in datasets:\n # dataset is already a dict, no need to call .dict()\n dataset_dict = dataset\n dataset_id = dataset_dict.get('id')\n \n # Get last task status\n last_task = self._get_last_task_for_resource(\n f\"dataset-{dataset_id}\", \n tasks\n )\n dataset_dict['last_task'] = last_task\n \n result.append(dataset_dict)\n \n logger.info(f\"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status\")\n return result\n # [/DEF:get_datasets_with_status:Function]\n \n # [DEF:get_activity_summary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get summary of active and recent tasks for the activity indicator\n # @PRE: tasks is a list of Task objects\n # @POST: Returns summary with active_count and recent_tasks\n # @PARAM: tasks (List[Task]) - List of tasks to summarize\n # @RETURN: Dict - Activity summary\n # @RELATION: CALLS ->[_extract_resource_name_from_task]\n # @RELATION: CALLS ->[_extract_resource_type_from_task]\n def get_activity_summary(self, tasks: List[Task]) -> Dict[str, Any]:\n with belief_scope(\"get_activity_summary\"):\n # Count active (RUNNING, WAITING_INPUT) tasks\n active_tasks = [\n t for t in tasks \n if t.status in ['RUNNING', 'WAITING_INPUT']\n ]\n \n # Get recent tasks (last 5)\n recent_tasks = sorted(\n tasks, \n key=lambda t: t.created_at, \n reverse=True\n )[:5]\n \n # Format recent tasks for frontend\n recent_tasks_formatted = []\n for task in recent_tasks:\n resource_name = self._extract_resource_name_from_task(task)\n recent_tasks_formatted.append({\n 'task_id': str(task.id),\n 'resource_name': resource_name,\n 'resource_type': self._extract_resource_type_from_task(task),\n 'status': task.status,\n 'started_at': task.created_at.isoformat() if task.created_at else None\n })\n \n return {\n 'active_count': len(active_tasks),\n 'recent_tasks': recent_tasks_formatted\n }\n # [/DEF:get_activity_summary:Function]\n \n # [DEF:_get_git_status_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get Git sync status for a dashboard\n # @PRE: dashboard_id is a valid integer\n # @POST: Returns git status or None if no repo exists\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @RETURN: Optional[Dict] - Git status with branch and sync_status\n # @RELATION: CALLS ->[get_repo]\n def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:\n try:\n repo = self.git_service.get_repo(dashboard_id)\n if not repo:\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n \n # Check if there are uncommitted changes\n try:\n # Get current branch\n branch = repo.active_branch.name\n \n # Check for uncommitted changes\n is_dirty = repo.is_dirty()\n has_changes_for_commit = repo.is_dirty(untracked_files=True)\n \n # Check for unpushed commits\n unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0\n \n if is_dirty or unpushed > 0:\n sync_status = 'DIFF'\n else:\n sync_status = 'OK'\n \n return {\n 'branch': branch,\n 'sync_status': sync_status,\n 'has_repo': True,\n 'has_changes_for_commit': has_changes_for_commit\n }\n except Exception:\n logger.warning(f\"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}\")\n return {\n 'branch': None,\n 'sync_status': 'ERROR',\n 'has_repo': True,\n 'has_changes_for_commit': False\n }\n except Exception:\n # No repo exists for this dashboard\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n # [/DEF:_get_git_status_for_dashboard:Function]\n \n # [DEF:_get_last_task_for_resource:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get the most recent task for a specific resource\n # @PRE: resource_id is a valid string\n # @POST: Returns task summary or None if no tasks found\n # @PARAM: resource_id (str) - The resource identifier (e.g., \"dashboard-123\")\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n def _get_last_task_for_resource(\n self,\n resource_id: str,\n tasks: Optional[List[Task]] = None\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n \n # Filter tasks for this resource\n resource_tasks = []\n for task in tasks:\n params = task.params or {}\n if params.get('resource_id') == resource_id:\n resource_tasks.append(task)\n \n if not resource_tasks:\n return None\n \n # Get most recent task with timezone-safe comparison.\n last_task = max(\n resource_tasks,\n key=lambda t: self._normalize_datetime_for_compare(getattr(t, \"created_at\", None)),\n )\n \n return {\n 'task_id': str(last_task.id),\n 'status': last_task.status\n }\n # [/DEF:_get_last_task_for_resource:Function]\n \n # [DEF:_extract_resource_name_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource name from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource name or task ID\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource name or fallback\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_name_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_name', f\"Task {task.id}\")\n # [/DEF:_extract_resource_name_from_task:Function]\n \n # [DEF:_extract_resource_type_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource type from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource type or 'unknown'\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource type\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_type_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_type', 'unknown')\n # [/DEF:_extract_resource_type_from_task:Function]\n# [/DEF:ResourceService:Class]\n" + "body": "# [DEF:ResourceService:Class]\n# @COMPLEXITY: 3\n# @PURPOSE: Provides centralized access to resource data with enhanced metadata\n# @RELATION: DEPENDS_ON ->[SupersetClient]\n# @RELATION: DEPENDS_ON ->[GitService]\nclass ResourceService:\n \n # [DEF:ResourceService_init:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initialize the resource service with dependencies\n # @PRE: None\n # @POST: ResourceService is ready to fetch resources\n def __init__(self):\n with belief_scope(\"ResourceService.__init__\"):\n self.git_service = GitService()\n log.reason(\"Initialized ResourceService\")\n # [/DEF:ResourceService_init:Function]\n \n # [DEF:get_dashboards_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch dashboards from environment with Git status and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of dashboards with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Dashboards with git_status and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummary]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_dashboards_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary(require_slug=require_slug)\n \n # Enhance each dashboard with Git status and task status\n result = []\n for dashboard in dashboards:\n # dashboard is already a dict, no need to call .dict()\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get('id')\n \n # Git status can be skipped for list endpoints and loaded lazily on UI side.\n if include_git_status:\n git_status = self._get_git_status_for_dashboard(dashboard_id)\n dashboard_dict['git_status'] = git_status\n else:\n dashboard_dict['git_status'] = None\n \n # Show status of the latest LLM validation for this dashboard.\n last_task = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n dashboard_dict['last_task'] = last_task\n \n result.append(dashboard_dict)\n \n log.reflect(f\"Fetched {len(result)} dashboards with status\")\n return result\n # [/DEF:get_dashboards_with_status:Function]\n\n # [DEF:get_dashboards_page_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.\n # @PRE: env is valid; page >= 1; page_size > 0.\n # @POST: Returns page items plus total counters without scanning all pages locally.\n # @PARAM: env (Environment) - Source environment.\n # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status.\n # @PARAM: page (int) - 1-based page number.\n # @PARAM: page_size (int) - Page size.\n # @RETURN: Dict[str, Any] - {\"dashboards\": List[Dict], \"total\": int, \"total_pages\": int}\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummaryPage]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_page_with_status(\n self,\n env: Any,\n tasks: Optional[List[Task]] = None,\n page: int = 1,\n page_size: int = 10,\n search: Optional[str] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\n \"get_dashboards_page_with_status\",\n f\"env={env.id}, page={page}, page_size={page_size}, search={search}\",\n ):\n client = SupersetClient(env)\n total, dashboards_page = client.get_dashboards_summary_page(\n page=page,\n page_size=page_size,\n search=search,\n require_slug=require_slug,\n )\n\n result = []\n for dashboard in dashboards_page:\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get(\"id\")\n\n if include_git_status:\n dashboard_dict[\"git_status\"] = self._get_git_status_for_dashboard(dashboard_id)\n else:\n dashboard_dict[\"git_status\"] = None\n\n dashboard_dict[\"last_task\"] = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n result.append(dashboard_dict)\n\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n log.reflect(f\"Fetched dashboards page {page}/{total_pages} ({len(result)} items, total={total})\")\n return {\n \"dashboards\": result,\n \"total\": total,\n \"total_pages\": total_pages,\n }\n # [/DEF:get_dashboards_page_with_status:Function]\n\n # [DEF:_get_last_llm_task_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment\n # @PRE: dashboard_id is a valid integer identifier\n # @POST: Returns the newest llm_dashboard_validation task summary or None\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @PARAM: env_id (Optional[str]) - Environment ID to match task params\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n # @RELATION: CALLS ->[_normalize_validation_status]\n # @RELATION: CALLS ->[_normalize_task_status]\n def _get_last_llm_task_for_dashboard(\n self,\n dashboard_id: int,\n env_id: Optional[str],\n tasks: Optional[List[Task]] = None,\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n\n dashboard_id_str = str(dashboard_id)\n matched_tasks = []\n\n for task in tasks:\n if getattr(task, \"plugin_id\", None) != \"llm_dashboard_validation\":\n continue\n\n params = getattr(task, \"params\", {}) or {}\n if str(params.get(\"dashboard_id\")) != dashboard_id_str:\n continue\n\n if env_id is not None:\n task_env = params.get(\"environment_id\") or params.get(\"env\")\n if str(task_env) != str(env_id):\n continue\n\n matched_tasks.append(task)\n\n if not matched_tasks:\n return None\n\n def _task_time(task_obj: Any) -> datetime:\n raw_time = (\n getattr(task_obj, \"started_at\", None)\n or getattr(task_obj, \"finished_at\", None)\n or getattr(task_obj, \"created_at\", None)\n )\n return self._normalize_datetime_for_compare(raw_time)\n\n projected_tasks = []\n for task in matched_tasks:\n raw_result = getattr(task, \"result\", None)\n validation_status = None\n if isinstance(raw_result, dict):\n validation_status = self._normalize_validation_status(raw_result.get(\"status\"))\n projected_tasks.append(\n (\n task,\n validation_status,\n _task_time(task),\n )\n )\n\n projected_tasks.sort(key=lambda item: item[2], reverse=True)\n latest_task, latest_validation_status, _ = projected_tasks[0]\n decisive_task = next(\n (\n item for item in projected_tasks\n if item[1] in {\"PASS\", \"WARN\", \"FAIL\"}\n ),\n None,\n )\n validation_status = latest_validation_status\n if validation_status == \"UNKNOWN\" and decisive_task is not None:\n validation_status = decisive_task[1]\n\n return {\n \"task_id\": str(getattr(latest_task, \"id\", \"\")),\n \"status\": self._normalize_task_status(getattr(latest_task, \"status\", \"\")),\n \"validation_status\": validation_status,\n }\n # [/DEF:_get_last_llm_task_for_dashboard:Function]\n\n # [DEF:_normalize_task_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections\n # @PRE: raw_status can be enum or string\n # @POST: Returns uppercase status without enum class prefix\n # @PARAM: raw_status (Any) - Raw task status object/value\n # @RETURN: str - Normalized status token\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_task_status(self, raw_status: Any) -> str:\n if raw_status is None:\n return \"\"\n value = getattr(raw_status, \"value\", raw_status)\n status_text = str(value).strip()\n if \".\" in status_text:\n status_text = status_text.split(\".\")[-1]\n return status_text.upper()\n # [/DEF:_normalize_task_status:Function]\n\n # [DEF:_normalize_validation_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN\n # @PRE: raw_status can be any scalar type\n # @POST: Returns normalized validation status token or None\n # @PARAM: raw_status (Any) - Raw validation status from task result\n # @RETURN: Optional[str] - PASS|FAIL|WARN|UNKNOWN\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n def _normalize_validation_status(self, raw_status: Any) -> Optional[str]:\n if raw_status is None:\n return None\n status_text = str(raw_status).strip().upper()\n if status_text in {\"PASS\", \"FAIL\", \"WARN\"}:\n return status_text\n return \"UNKNOWN\"\n # [/DEF:_normalize_validation_status:Function]\n\n # [DEF:_normalize_datetime_for_compare:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons.\n # @PRE: value may be datetime or any scalar.\n # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime.\n # @PARAM: value (Any) - Candidate datetime-like value.\n # @RETURN: datetime - UTC-aware comparable datetime.\n # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard]\n # @RELATION: USED_BY ->[_get_last_task_for_resource]\n def _normalize_datetime_for_compare(self, value: Any) -> datetime:\n if isinstance(value, datetime):\n if value.tzinfo is None:\n return value.replace(tzinfo=timezone.utc)\n return value.astimezone(timezone.utc)\n return datetime.min.replace(tzinfo=timezone.utc)\n # [/DEF:_normalize_datetime_for_compare:Function]\n \n # [DEF:get_datasets_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch datasets from environment with mapping progress and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of datasets with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary]\n # @RELATION: CALLS ->[_get_last_task_for_resource]\n async def get_datasets_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_datasets_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n datasets = client.get_datasets_summary()\n \n # Enhance each dataset with task status\n result = []\n for dataset in datasets:\n # dataset is already a dict, no need to call .dict()\n dataset_dict = dataset\n dataset_id = dataset_dict.get('id')\n \n # Get last task status\n last_task = self._get_last_task_for_resource(\n f\"dataset-{dataset_id}\", \n tasks\n )\n dataset_dict['last_task'] = last_task\n \n result.append(dataset_dict)\n \n log.reflect(f\"Fetched {len(result)} datasets with status\")\n return result\n # [/DEF:get_datasets_with_status:Function]\n \n # [DEF:get_activity_summary:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get summary of active and recent tasks for the activity indicator\n # @PRE: tasks is a list of Task objects\n # @POST: Returns summary with active_count and recent_tasks\n # @PARAM: tasks (List[Task]) - List of tasks to summarize\n # @RETURN: Dict - Activity summary\n # @RELATION: CALLS ->[_extract_resource_name_from_task]\n # @RELATION: CALLS ->[_extract_resource_type_from_task]\n def get_activity_summary(self, tasks: List[Task]) -> Dict[str, Any]:\n with belief_scope(\"get_activity_summary\"):\n # Count active (RUNNING, WAITING_INPUT) tasks\n active_tasks = [\n t for t in tasks \n if t.status in ['RUNNING', 'WAITING_INPUT']\n ]\n \n # Get recent tasks (last 5)\n recent_tasks = sorted(\n tasks, \n key=lambda t: t.created_at, \n reverse=True\n )[:5]\n \n # Format recent tasks for frontend\n recent_tasks_formatted = []\n for task in recent_tasks:\n resource_name = self._extract_resource_name_from_task(task)\n recent_tasks_formatted.append({\n 'task_id': str(task.id),\n 'resource_name': resource_name,\n 'resource_type': self._extract_resource_type_from_task(task),\n 'status': task.status,\n 'started_at': task.created_at.isoformat() if task.created_at else None\n })\n \n return {\n 'active_count': len(active_tasks),\n 'recent_tasks': recent_tasks_formatted\n }\n # [/DEF:get_activity_summary:Function]\n \n # [DEF:_get_git_status_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get Git sync status for a dashboard\n # @PRE: dashboard_id is a valid integer\n # @POST: Returns git status or None if no repo exists\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @RETURN: Optional[Dict] - Git status with branch and sync_status\n # @RELATION: CALLS ->[get_repo]\n def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:\n try:\n repo = self.git_service.get_repo(dashboard_id)\n if not repo:\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n \n # Check if there are uncommitted changes\n try:\n # Get current branch\n branch = repo.active_branch.name\n \n # Check for uncommitted changes\n is_dirty = repo.is_dirty()\n has_changes_for_commit = repo.is_dirty(untracked_files=True)\n \n # Check for unpushed commits\n unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0\n \n if is_dirty or unpushed > 0:\n sync_status = 'DIFF'\n else:\n sync_status = 'OK'\n \n return {\n 'branch': branch,\n 'sync_status': sync_status,\n 'has_repo': True,\n 'has_changes_for_commit': has_changes_for_commit\n }\n except Exception:\n log.explore(f\"Failed to get git status for dashboard {dashboard_id}\", error=\"Git status check failed\")\n return {\n 'branch': None,\n 'sync_status': 'ERROR',\n 'has_repo': True,\n 'has_changes_for_commit': False\n }\n except Exception:\n # No repo exists for this dashboard\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n # [/DEF:_get_git_status_for_dashboard:Function]\n \n # [DEF:_get_last_task_for_resource:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get the most recent task for a specific resource\n # @PRE: resource_id is a valid string\n # @POST: Returns task summary or None if no tasks found\n # @PARAM: resource_id (str) - The resource identifier (e.g., \"dashboard-123\")\n # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search\n # @RETURN: Optional[Dict] - Task summary with task_id and status\n # @RELATION: CALLS ->[_normalize_datetime_for_compare]\n def _get_last_task_for_resource(\n self,\n resource_id: str,\n tasks: Optional[List[Task]] = None\n ) -> Optional[Dict[str, Any]]:\n if not tasks:\n return None\n \n # Filter tasks for this resource\n resource_tasks = []\n for task in tasks:\n params = task.params or {}\n if params.get('resource_id') == resource_id:\n resource_tasks.append(task)\n \n if not resource_tasks:\n return None\n \n # Get most recent task with timezone-safe comparison.\n last_task = max(\n resource_tasks,\n key=lambda t: self._normalize_datetime_for_compare(getattr(t, \"created_at\", None)),\n )\n \n return {\n 'task_id': str(last_task.id),\n 'status': last_task.status\n }\n # [/DEF:_get_last_task_for_resource:Function]\n \n # [DEF:_extract_resource_name_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource name from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource name or task ID\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource name or fallback\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_name_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_name', f\"Task {task.id}\")\n # [/DEF:_extract_resource_name_from_task:Function]\n \n # [DEF:_extract_resource_type_from_task:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Extract resource type from task params\n # @PRE: task is a valid Task object\n # @POST: Returns resource type or 'unknown'\n # @PARAM: task (Task) - The task to extract from\n # @RETURN: str - Resource type\n # @RELATION: USED_BY ->[get_activity_summary]\n def _extract_resource_type_from_task(self, task: Task) -> str:\n params = task.params or {}\n return params.get('resource_type', 'unknown')\n # [/DEF:_extract_resource_type_from_task:Function]\n# [/DEF:ResourceService:Class]\n" }, { "contract_id": "ResourceService_init", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 28, - "end_line": 37, + "start_line": 31, + "end_line": 40, "tier": "TIER_1", "complexity": 1, "metadata": { @@ -92721,14 +93730,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:ResourceService_init:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initialize the resource service with dependencies\n # @PRE: None\n # @POST: ResourceService is ready to fetch resources\n def __init__(self):\n with belief_scope(\"ResourceService.__init__\"):\n self.git_service = GitService()\n logger.info(\"[ResourceService][Action] Initialized ResourceService\")\n # [/DEF:ResourceService_init:Function]\n" + "body": " # [DEF:ResourceService_init:Function]\n # @COMPLEXITY: 1\n # @PURPOSE: Initialize the resource service with dependencies\n # @PRE: None\n # @POST: ResourceService is ready to fetch resources\n def __init__(self):\n with belief_scope(\"ResourceService.__init__\"):\n self.git_service = GitService()\n log.reason(\"Initialized ResourceService\")\n # [/DEF:ResourceService_init:Function]\n" }, { "contract_id": "get_dashboards_with_status", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 39, - "end_line": 87, + "start_line": 42, + "end_line": 90, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -92792,14 +93801,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_dashboards_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch dashboards from environment with Git status and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of dashboards with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Dashboards with git_status and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummary]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_dashboards_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary(require_slug=require_slug)\n \n # Enhance each dashboard with Git status and task status\n result = []\n for dashboard in dashboards:\n # dashboard is already a dict, no need to call .dict()\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get('id')\n \n # Git status can be skipped for list endpoints and loaded lazily on UI side.\n if include_git_status:\n git_status = self._get_git_status_for_dashboard(dashboard_id)\n dashboard_dict['git_status'] = git_status\n else:\n dashboard_dict['git_status'] = None\n \n # Show status of the latest LLM validation for this dashboard.\n last_task = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n dashboard_dict['last_task'] = last_task\n \n result.append(dashboard_dict)\n \n logger.info(f\"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status\")\n return result\n # [/DEF:get_dashboards_with_status:Function]\n" + "body": " # [DEF:get_dashboards_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch dashboards from environment with Git status and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of dashboards with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Dashboards with git_status and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummary]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_dashboards_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n dashboards = client.get_dashboards_summary(require_slug=require_slug)\n \n # Enhance each dashboard with Git status and task status\n result = []\n for dashboard in dashboards:\n # dashboard is already a dict, no need to call .dict()\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get('id')\n \n # Git status can be skipped for list endpoints and loaded lazily on UI side.\n if include_git_status:\n git_status = self._get_git_status_for_dashboard(dashboard_id)\n dashboard_dict['git_status'] = git_status\n else:\n dashboard_dict['git_status'] = None\n \n # Show status of the latest LLM validation for this dashboard.\n last_task = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n dashboard_dict['last_task'] = last_task\n \n result.append(dashboard_dict)\n \n log.reflect(f\"Fetched {len(result)} dashboards with status\")\n return result\n # [/DEF:get_dashboards_with_status:Function]\n" }, { "contract_id": "get_dashboards_page_with_status", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 89, - "end_line": 154, + "start_line": 92, + "end_line": 151, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -92863,14 +93872,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_dashboards_page_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.\n # @PRE: env is valid; page >= 1; page_size > 0.\n # @POST: Returns page items plus total counters without scanning all pages locally.\n # @PARAM: env (Environment) - Source environment.\n # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status.\n # @PARAM: page (int) - 1-based page number.\n # @PARAM: page_size (int) - Page size.\n # @RETURN: Dict[str, Any] - {\"dashboards\": List[Dict], \"total\": int, \"total_pages\": int}\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummaryPage]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_page_with_status(\n self,\n env: Any,\n tasks: Optional[List[Task]] = None,\n page: int = 1,\n page_size: int = 10,\n search: Optional[str] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\n \"get_dashboards_page_with_status\",\n f\"env={env.id}, page={page}, page_size={page_size}, search={search}\",\n ):\n client = SupersetClient(env)\n total, dashboards_page = client.get_dashboards_summary_page(\n page=page,\n page_size=page_size,\n search=search,\n require_slug=require_slug,\n )\n\n result = []\n for dashboard in dashboards_page:\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get(\"id\")\n\n if include_git_status:\n dashboard_dict[\"git_status\"] = self._get_git_status_for_dashboard(dashboard_id)\n else:\n dashboard_dict[\"git_status\"] = None\n\n dashboard_dict[\"last_task\"] = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n result.append(dashboard_dict)\n\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n logger.info(\n \"[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)\",\n page,\n total_pages,\n len(result),\n total,\n )\n return {\n \"dashboards\": result,\n \"total\": total,\n \"total_pages\": total_pages,\n }\n # [/DEF:get_dashboards_page_with_status:Function]\n" + "body": " # [DEF:get_dashboards_page_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata.\n # @PRE: env is valid; page >= 1; page_size > 0.\n # @POST: Returns page items plus total counters without scanning all pages locally.\n # @PARAM: env (Environment) - Source environment.\n # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status.\n # @PARAM: page (int) - 1-based page number.\n # @PARAM: page_size (int) - Page size.\n # @RETURN: Dict[str, Any] - {\"dashboards\": List[Dict], \"total\": int, \"total_pages\": int}\n # @RELATION: CALLS -> [SupersetClientGetDashboardsSummaryPage]\n # @RELATION: CALLS ->[_get_git_status_for_dashboard]\n # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard]\n async def get_dashboards_page_with_status(\n self,\n env: Any,\n tasks: Optional[List[Task]] = None,\n page: int = 1,\n page_size: int = 10,\n search: Optional[str] = None,\n include_git_status: bool = True,\n require_slug: bool = False,\n ) -> Dict[str, Any]:\n with belief_scope(\n \"get_dashboards_page_with_status\",\n f\"env={env.id}, page={page}, page_size={page_size}, search={search}\",\n ):\n client = SupersetClient(env)\n total, dashboards_page = client.get_dashboards_summary_page(\n page=page,\n page_size=page_size,\n search=search,\n require_slug=require_slug,\n )\n\n result = []\n for dashboard in dashboards_page:\n dashboard_dict = dashboard\n dashboard_id = dashboard_dict.get(\"id\")\n\n if include_git_status:\n dashboard_dict[\"git_status\"] = self._get_git_status_for_dashboard(dashboard_id)\n else:\n dashboard_dict[\"git_status\"] = None\n\n dashboard_dict[\"last_task\"] = self._get_last_llm_task_for_dashboard(\n dashboard_id,\n env.id,\n tasks,\n )\n result.append(dashboard_dict)\n\n total_pages = (total + page_size - 1) // page_size if total > 0 else 1\n log.reflect(f\"Fetched dashboards page {page}/{total_pages} ({len(result)} items, total={total})\")\n return {\n \"dashboards\": result,\n \"total\": total,\n \"total_pages\": total_pages,\n }\n # [/DEF:get_dashboards_page_with_status:Function]\n" }, { "contract_id": "_get_last_llm_task_for_dashboard", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 156, - "end_line": 238, + "start_line": 153, + "end_line": 235, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -92940,8 +93949,8 @@ "contract_id": "_normalize_task_status", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 240, - "end_line": 256, + "start_line": 237, + "end_line": 253, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93016,8 +94025,8 @@ "contract_id": "_normalize_validation_status", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 258, - "end_line": 273, + "start_line": 255, + "end_line": 270, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93092,8 +94101,8 @@ "contract_id": "_normalize_datetime_for_compare", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 275, - "end_line": 290, + "start_line": 272, + "end_line": 287, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93191,8 +94200,8 @@ "contract_id": "get_datasets_with_status", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 292, - "end_line": 329, + "start_line": 289, + "end_line": 326, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93250,14 +94259,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:get_datasets_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch datasets from environment with mapping progress and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of datasets with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary]\n # @RELATION: CALLS ->[_get_last_task_for_resource]\n async def get_datasets_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_datasets_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n datasets = client.get_datasets_summary()\n \n # Enhance each dataset with task status\n result = []\n for dataset in datasets:\n # dataset is already a dict, no need to call .dict()\n dataset_dict = dataset\n dataset_id = dataset_dict.get('id')\n \n # Get last task status\n last_task = self._get_last_task_for_resource(\n f\"dataset-{dataset_id}\", \n tasks\n )\n dataset_dict['last_task'] = last_task\n \n result.append(dataset_dict)\n \n logger.info(f\"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status\")\n return result\n # [/DEF:get_datasets_with_status:Function]\n" + "body": " # [DEF:get_datasets_with_status:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Fetch datasets from environment with mapping progress and last task status\n # @PRE: env is a valid Environment object\n # @POST: Returns list of datasets with enhanced metadata\n # @PARAM: env (Environment) - The environment to fetch from\n # @PARAM: tasks (List[Task]) - List of tasks to check for status\n # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields\n # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary]\n # @RELATION: CALLS ->[_get_last_task_for_resource]\n async def get_datasets_with_status(\n self, \n env: Any, \n tasks: Optional[List[Task]] = None\n ) -> List[Dict[str, Any]]:\n with belief_scope(\"get_datasets_with_status\", f\"env={env.id}\"):\n client = SupersetClient(env)\n datasets = client.get_datasets_summary()\n \n # Enhance each dataset with task status\n result = []\n for dataset in datasets:\n # dataset is already a dict, no need to call .dict()\n dataset_dict = dataset\n dataset_id = dataset_dict.get('id')\n \n # Get last task status\n last_task = self._get_last_task_for_resource(\n f\"dataset-{dataset_id}\", \n tasks\n )\n dataset_dict['last_task'] = last_task\n \n result.append(dataset_dict)\n \n log.reflect(f\"Fetched {len(result)} datasets with status\")\n return result\n # [/DEF:get_datasets_with_status:Function]\n" }, { "contract_id": "get_activity_summary", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 331, - "end_line": 371, + "start_line": 328, + "end_line": 368, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93321,8 +94330,8 @@ "contract_id": "_get_git_status_for_dashboard", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 373, - "end_line": 431, + "start_line": 370, + "end_line": 428, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93374,14 +94383,14 @@ } ], "anchor_syntax": "def", - "body": " # [DEF:_get_git_status_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get Git sync status for a dashboard\n # @PRE: dashboard_id is a valid integer\n # @POST: Returns git status or None if no repo exists\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @RETURN: Optional[Dict] - Git status with branch and sync_status\n # @RELATION: CALLS ->[get_repo]\n def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:\n try:\n repo = self.git_service.get_repo(dashboard_id)\n if not repo:\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n \n # Check if there are uncommitted changes\n try:\n # Get current branch\n branch = repo.active_branch.name\n \n # Check for uncommitted changes\n is_dirty = repo.is_dirty()\n has_changes_for_commit = repo.is_dirty(untracked_files=True)\n \n # Check for unpushed commits\n unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0\n \n if is_dirty or unpushed > 0:\n sync_status = 'DIFF'\n else:\n sync_status = 'OK'\n \n return {\n 'branch': branch,\n 'sync_status': sync_status,\n 'has_repo': True,\n 'has_changes_for_commit': has_changes_for_commit\n }\n except Exception:\n logger.warning(f\"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}\")\n return {\n 'branch': None,\n 'sync_status': 'ERROR',\n 'has_repo': True,\n 'has_changes_for_commit': False\n }\n except Exception:\n # No repo exists for this dashboard\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n # [/DEF:_get_git_status_for_dashboard:Function]\n" + "body": " # [DEF:_get_git_status_for_dashboard:Function]\n # @COMPLEXITY: 3\n # @PURPOSE: Get Git sync status for a dashboard\n # @PRE: dashboard_id is a valid integer\n # @POST: Returns git status or None if no repo exists\n # @PARAM: dashboard_id (int) - The dashboard ID\n # @RETURN: Optional[Dict] - Git status with branch and sync_status\n # @RELATION: CALLS ->[get_repo]\n def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:\n try:\n repo = self.git_service.get_repo(dashboard_id)\n if not repo:\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n \n # Check if there are uncommitted changes\n try:\n # Get current branch\n branch = repo.active_branch.name\n \n # Check for uncommitted changes\n is_dirty = repo.is_dirty()\n has_changes_for_commit = repo.is_dirty(untracked_files=True)\n \n # Check for unpushed commits\n unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0\n \n if is_dirty or unpushed > 0:\n sync_status = 'DIFF'\n else:\n sync_status = 'OK'\n \n return {\n 'branch': branch,\n 'sync_status': sync_status,\n 'has_repo': True,\n 'has_changes_for_commit': has_changes_for_commit\n }\n except Exception:\n log.explore(f\"Failed to get git status for dashboard {dashboard_id}\", error=\"Git status check failed\")\n return {\n 'branch': None,\n 'sync_status': 'ERROR',\n 'has_repo': True,\n 'has_changes_for_commit': False\n }\n except Exception:\n # No repo exists for this dashboard\n return {\n 'branch': None,\n 'sync_status': 'NO_REPO',\n 'has_repo': False,\n 'has_changes_for_commit': False\n }\n # [/DEF:_get_git_status_for_dashboard:Function]\n" }, { "contract_id": "_get_last_task_for_resource", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 433, - "end_line": 470, + "start_line": 430, + "end_line": 467, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93439,8 +94448,8 @@ "contract_id": "_extract_resource_name_from_task", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 472, - "end_line": 483, + "start_line": 469, + "end_line": 480, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -93515,8 +94524,8 @@ "contract_id": "_extract_resource_type_from_task", "contract_type": "Function", "file_path": "backend/src/services/resource_service.py", - "start_line": 485, - "end_line": 496, + "start_line": 482, + "end_line": 493, "tier": "TIER_2", "complexity": 3, "metadata": { @@ -100041,18 +101050,20 @@ "contract_type": "Module", "file_path": "backend/tests/test_logger.py", "start_line": 1, - "end_line": 230, + "end_line": 360, "tier": "TIER_2", "complexity": 3, "metadata": { "COMPLEXITY": 3, "INVARIANT": "All required log statements must correctly check the threshold.", "LAYER": "Logging (Tests)", - "PURPOSE": "Unit tests for the custom logger formatters and configuration context manager.", + "PURPOSE": "Unit tests for the custom logger CoT JSON formatters and configuration context manager.", "SEMANTICS": [ "logging", "tests", - "belief_state" + "belief_state", + "cot", + "json" ] }, "relations": [ @@ -100089,7 +101100,7 @@ } ], "anchor_syntax": "def", - "body": "# [DEF:TestLogger:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: logging, tests, belief_state\n# @PURPOSE: Unit tests for the custom logger formatters and configuration context manager.\n# @LAYER: Logging (Tests)\n# @RELATION: VERIFIES -> src/core/logger.py\n# @INVARIANT: All required log statements must correctly check the threshold.\n\nimport pytest\nfrom src.core.logger import (\n belief_scope,\n logger,\n configure_logger,\n get_task_log_level,\n should_log_task_level\n)\nfrom src.core.config_models import LoggingConfig\n\n\n# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level.\ndef test_belief_scope_logs_entry_action_exit_at_debug(caplog):\n \"\"\"Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"TestFunction\"):\n logger.info(\"Doing something important\")\n\n # Check that the logs contain the expected patterns\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[TestFunction][Entry]\" in msg for msg in log_messages), \"Entry log not found\"\n assert any(\"[TestFunction][Action] Doing something important\" in msg for msg in log_messages), \"Action log not found\"\n assert any(\"[TestFunction][Exit]\" in msg for msg in log_messages), \"Exit log not found\"\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]\n\n\n# [DEF:test_belief_scope_error_handling:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Coherence:Failed tag.\ndef test_belief_scope_error_handling(caplog):\n \"\"\"Test that belief_scope logs Coherence:Failed on exception.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with pytest.raises(ValueError):\n with belief_scope(\"FailingFunction\"):\n raise ValueError(\"Something went wrong\")\n\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[FailingFunction][Entry]\" in msg for msg in log_messages), f\"Entry log not found. Logs: {log_messages}\"\n assert any(\"[FailingFunction][COHERENCE:FAILED]\" in msg for msg in log_messages), f\"Failed coherence log not found. Logs: {log_messages}\"\n # Exit should not be logged on failure\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_error_handling:Function]\n\n\n# [DEF:test_belief_scope_success_coherence:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope logs Coherence:OK on success.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain Coherence:OK tag.\ndef test_belief_scope_success_coherence(caplog):\n \"\"\"Test that belief_scope logs Coherence:OK on success.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"SuccessFunction\"):\n pass\n\n log_messages = [record.message for record in caplog.records]\n\n assert any(\"[SuccessFunction][COHERENCE:OK]\" in msg for msg in log_messages), f\"Success coherence log not found. Logs: {log_messages}\"\n \n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_success_coherence:Function]\n\n\n# [DEF:test_belief_scope_not_visible_at_info:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.\n# @PRE: belief_scope is available. caplog fixture is used.\n# @POST: Entry/Exit/Coherence logs are not captured at INFO level.\ndef test_belief_scope_not_visible_at_info(caplog):\n \"\"\"Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.\"\"\"\n caplog.set_level(\"INFO\")\n\n with belief_scope(\"InfoLevelFunction\"):\n logger.info(\"Doing something important\")\n\n log_messages = [record.message for record in caplog.records]\n\n # Action log should be visible\n assert any(\"[InfoLevelFunction][Action] Doing something important\" in msg for msg in log_messages), \"Action log not found\"\n # Entry/Exit/Coherence should NOT be visible at INFO level\n assert not any(\"[InfoLevelFunction][Entry]\" in msg for msg in log_messages), \"Entry log should not be visible at INFO\"\n assert not any(\"[InfoLevelFunction][Exit]\" in msg for msg in log_messages), \"Exit log should not be visible at INFO\"\n assert not any(\"[InfoLevelFunction][Coherence:OK]\" in msg for msg in log_messages), \"Coherence log should not be visible at INFO\"\n# [/DEF:test_belief_scope_not_visible_at_info:Function]\n\n\n# [DEF:test_task_log_level_default:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that default task log level is INFO.\n# @PRE: None.\n# @POST: Default level is INFO.\ndef test_task_log_level_default():\n \"\"\"Test that default task log level is INFO.\"\"\"\n level = get_task_log_level()\n assert level == \"INFO\"\n# [/DEF:test_task_log_level_default:Function]\n\n\n# [DEF:test_should_log_task_level:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that should_log_task_level correctly filters log levels.\n# @PRE: None.\n# @POST: Filtering works correctly for all level combinations.\ndef test_should_log_task_level():\n \"\"\"Test that should_log_task_level correctly filters log levels.\"\"\"\n # Default level is INFO\n assert should_log_task_level(\"ERROR\") is True, \"ERROR should be logged at INFO threshold\"\n assert should_log_task_level(\"WARNING\") is True, \"WARNING should be logged at INFO threshold\"\n assert should_log_task_level(\"INFO\") is True, \"INFO should be logged at INFO threshold\"\n assert should_log_task_level(\"DEBUG\") is False, \"DEBUG should NOT be logged at INFO threshold\"\n# [/DEF:test_should_log_task_level:Function]\n\n\n# [DEF:test_configure_logger_task_log_level:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that configure_logger updates task_log_level.\n# @PRE: LoggingConfig is available.\n# @POST: task_log_level is updated correctly.\ndef test_configure_logger_task_log_level():\n \"\"\"Test that configure_logger updates task_log_level.\"\"\"\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n \n assert get_task_log_level() == \"DEBUG\", \"task_log_level should be DEBUG\"\n assert should_log_task_level(\"DEBUG\") is True, \"DEBUG should be logged at DEBUG threshold\"\n \n # Reset to INFO\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n assert get_task_log_level() == \"INFO\", \"task_log_level should be reset to INFO\"\n# [/DEF:test_configure_logger_task_log_level:Function]\n\n\n# [DEF:test_enable_belief_state_flag:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging.\n# @PRE: LoggingConfig is available. caplog fixture is used.\n# @POST: belief_scope logs are controlled by the flag.\ndef test_enable_belief_state_flag(caplog):\n \"\"\"Test that enable_belief_state flag controls belief_scope logging.\"\"\"\n # Disable belief state\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False\n )\n configure_logger(config)\n \n caplog.set_level(\"DEBUG\")\n \n with belief_scope(\"DisabledFunction\"):\n logger.info(\"Doing something\")\n \n log_messages = [record.message for record in caplog.records]\n \n # Entry and Exit should NOT be logged when disabled\n assert not any(\"[DisabledFunction][Entry]\" in msg for msg in log_messages), \"Entry should not be logged when disabled\"\n assert not any(\"[DisabledFunction][Exit]\" in msg for msg in log_messages), \"Exit should not be logged when disabled\"\n # Coherence:OK should still be logged (internal tracking)\n assert any(\"[DisabledFunction][COHERENCE:OK]\" in msg for msg in log_messages), \"Coherence should still be logged\"\n \n # Re-enable for other tests\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n# [/DEF:test_enable_belief_state_flag:Function]\n# [/DEF:TestLogger:Module]\n" + "body": "# [DEF:TestLogger:Module]\n# @COMPLEXITY: 3\n# @SEMANTICS: logging, tests, belief_state, cot, json\n# @PURPOSE: Unit tests for the custom logger CoT JSON formatters and configuration context manager.\n# @LAYER: Logging (Tests)\n# @RELATION: VERIFIES -> src/core/logger.py\n# @INVARIANT: All required log statements must correctly check the threshold.\n\nimport pytest\nimport logging\n\nfrom src.core.logger import (\n belief_scope,\n logger,\n configure_logger,\n get_task_log_level,\n should_log_task_level,\n CotJsonFormatter,\n)\nfrom src.core.config_models import LoggingConfig\n\n\n@pytest.fixture(autouse=True)\ndef reset_logger_state():\n \"\"\"Reset logger state before each test to avoid cross-test contamination.\"\"\"\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n # Also reset the logger level for caplog to work correctly\n logging.getLogger(\"superset_tools_app\").setLevel(logging.DEBUG)\n yield\n # Reset after test too\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n\n# [DEF:test_belief_scope_logs_reason_reflect_at_debug:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain REASON (entry) and REFLECT (coherence) markers.\ndef test_belief_scope_logs_reason_reflect_at_debug(caplog):\n \"\"\"Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"TestFunction\"):\n logger.info(\"Doing something important\")\n\n # Check that the records contain the expected markers\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction'\n ]\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction'\n ]\n info_records = [\n r for r in caplog.records\n if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()\n ]\n\n assert len(reason_records) >= 1, \"REASON marker not found for TestFunction\"\n assert len(reflect_records) >= 1, \"REFLECT marker not found for TestFunction\"\n assert len(info_records) >= 1, \"INFO log 'Doing something important' not found\"\n\n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_logs_reason_reflect_at_debug:Function]\n\n\n# [DEF:test_belief_scope_error_handling:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope logs EXPLORE marker on exception.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain EXPLORE marker with error context.\ndef test_belief_scope_error_handling(caplog):\n \"\"\"Test that belief_scope logs EXPLORE marker on exception.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with pytest.raises(ValueError):\n with belief_scope(\"FailingFunction\"):\n raise ValueError(\"Something went wrong\")\n\n # Check that an EXPLORE marker was emitted\n explore_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'EXPLORE'\n and getattr(r, 'src', None) == 'FailingFunction'\n ]\n\n assert len(explore_records) >= 1, f\"EXPLORE marker not found for FailingFunction. Logs: {[(r.levelname, getattr(r, 'marker', ''), r.getMessage()) for r in caplog.records]}\"\n assert 'Something went wrong' in explore_records[0].getMessage() or \\\n 'Something went wrong' in str(getattr(explore_records[0], 'error', ''))\n\n # Reset to INFO\n config = LoggingConfig(level=\"INFO\", task_log_level=\"INFO\", enable_belief_state=True)\n configure_logger(config)\n# [/DEF:test_belief_scope_error_handling:Function]\n\n\n# [DEF:test_belief_scope_success_coherence:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope logs REFLECT marker on success.\n# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.\n# @POST: Logs are verified to contain REFLECT marker.\ndef test_belief_scope_success_coherence(caplog):\n \"\"\"Test that belief_scope logs REFLECT marker on success.\"\"\"\n # Configure logger to DEBUG level\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"SuccessFunction\"):\n pass\n\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT'\n and getattr(r, 'src', None) == 'SuccessFunction'\n ]\n\n assert len(reflect_records) >= 1, f\"REFLECT marker not found for SuccessFunction. Logs: {[(r.levelname, getattr(r, 'marker', ''), r.getMessage()) for r in caplog.records]}\"\n assert 'Coherence OK' in reflect_records[0].getMessage() or \\\n getattr(reflect_records[0], 'intent', '') == 'Coherence OK'\n\n\n# [/DEF:test_belief_scope_success_coherence:Function]\n\n\n# [DEF:test_belief_scope_reason_not_visible_at_info:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.\n# @PRE: belief_scope is available. caplog fixture is used.\n# @POST: REASON/REFLECT markers are not captured at INFO level.\ndef test_belief_scope_reason_not_visible_at_info(caplog):\n \"\"\"Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.\"\"\"\n caplog.set_level(\"INFO\")\n\n with belief_scope(\"InfoLevelFunction\"):\n logger.info(\"Doing something important\")\n\n # The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction'\n ]\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction'\n ]\n\n assert len(reason_records) == 0, \"REASON marker should not be visible at INFO\"\n assert len(reflect_records) == 0, \"REFLECT marker should not be visible at INFO\"\n\n # But the INFO-level message should be visible\n info_records = [\n r for r in caplog.records\n if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()\n ]\n assert len(info_records) >= 1, \"INFO log 'Doing something important' should be visible\"\n# [/DEF:test_belief_scope_reason_not_visible_at_info:Function]\n\n\n# [DEF:test_task_log_level_default:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that default task log level is INFO.\n# @PRE: None.\n# @POST: Default level is INFO.\ndef test_task_log_level_default():\n \"\"\"Test that default task log level is INFO.\"\"\"\n level = get_task_log_level()\n assert level == \"INFO\"\n# [/DEF:test_task_log_level_default:Function]\n\n\n# [DEF:test_should_log_task_level:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that should_log_task_level correctly filters log levels.\n# @PRE: None.\n# @POST: Filtering works correctly for all level combinations.\ndef test_should_log_task_level():\n \"\"\"Test that should_log_task_level correctly filters log levels.\"\"\"\n # Default level is INFO\n assert should_log_task_level(\"ERROR\") is True, \"ERROR should be logged at INFO threshold\"\n assert should_log_task_level(\"WARNING\") is True, \"WARNING should be logged at INFO threshold\"\n assert should_log_task_level(\"INFO\") is True, \"INFO should be logged at INFO threshold\"\n assert should_log_task_level(\"DEBUG\") is False, \"DEBUG should NOT be logged at INFO threshold\"\n# [/DEF:test_should_log_task_level:Function]\n\n\n# [DEF:test_configure_logger_task_log_level:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that configure_logger updates task_log_level.\n# @PRE: LoggingConfig is available.\n# @POST: task_log_level is updated correctly.\ndef test_configure_logger_task_log_level():\n \"\"\"Test that configure_logger updates task_log_level.\"\"\"\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n\n assert get_task_log_level() == \"DEBUG\", \"task_log_level should be DEBUG\"\n assert should_log_task_level(\"DEBUG\") is True, \"DEBUG should be logged at DEBUG threshold\"\n\n # Reset to INFO\n config = LoggingConfig(\n level=\"INFO\",\n task_log_level=\"INFO\",\n enable_belief_state=True\n )\n configure_logger(config)\n assert get_task_log_level() == \"INFO\", \"task_log_level should be reset to INFO\"\n# [/DEF:test_configure_logger_task_log_level:Function]\n\n\n# [DEF:test_enable_belief_state_flag:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that enable_belief_state flag controls belief_scope entry logging.\n# @PRE: LoggingConfig is available. caplog fixture is used.\n# @POST: REASON entry marker suppressed when disabled; REFLECT coherence still logged.\ndef test_enable_belief_state_flag(caplog):\n \"\"\"Test that enable_belief_state flag controls belief_scope REASON entry logging.\"\"\"\n # Disable belief state\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=False\n )\n configure_logger(config)\n\n caplog.set_level(\"DEBUG\")\n\n with belief_scope(\"DisabledFunction\"):\n logger.info(\"Doing something\")\n\n # REASON entry marker should NOT be logged when disabled\n reason_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction'\n ]\n assert len(reason_records) == 0, \"REASON entry should not be logged when disabled\"\n\n # REFLECT coherence marker should still be logged (internal tracking)\n reflect_records = [\n r for r in caplog.records\n if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction'\n ]\n assert len(reflect_records) >= 1, \"REFLECT coherence should still be logged\"\n\n # Re-enable for other tests\n config = LoggingConfig(\n level=\"DEBUG\",\n task_log_level=\"DEBUG\",\n enable_belief_state=True\n )\n configure_logger(config)\n# [/DEF:test_enable_belief_state_flag:Function]\n\n# [DEF:test_cot_json_formatter_output:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that CotJsonFormatter produces valid JSON with expected fields.\ndef test_cot_json_formatter_output():\n \"\"\"Test that CotJsonFormatter produces valid JSON with expected fields.\"\"\"\n import json\n from datetime import datetime, timezone\n\n formatter = CotJsonFormatter()\n\n # Create a LogRecord with structured extra data\n record = logging.LogRecord(\n name=\"test.module\",\n level=logging.INFO,\n pathname=\"/fake/path.py\",\n lineno=42,\n msg=\"Test action\",\n args=(),\n exc_info=None,\n )\n record.marker = \"REASON\"\n record.intent = \"Test action\"\n record.src = \"TestModule\"\n record.payload = {\"key\": \"value\"}\n\n output = formatter.format(record)\n parsed = json.loads(output)\n\n assert parsed[\"level\"] == \"INFO\"\n assert parsed[\"marker\"] == \"REASON\"\n assert parsed[\"intent\"] == \"Test action\"\n assert parsed[\"src\"] == \"TestModule\"\n assert parsed[\"payload\"] == {\"key\": \"value\"}\n assert \"ts\" in parsed\n assert \"trace_id\" in parsed\n# [/DEF:test_cot_json_formatter_output:Function]\n\n# [DEF:test_cot_json_formatter_plain_message:Function]\n# @RELATION: BINDS_TO -> TestLogger\n# @PURPOSE: Test that CotJsonFormatter wraps plain messages (no extra) with default marker.\ndef test_cot_json_formatter_plain_message():\n \"\"\"Test that CotJsonFormatter wraps plain messages with default marker.\"\"\"\n import json\n\n formatter = CotJsonFormatter()\n\n # Create a LogRecord WITHOUT extra data (plain message)\n record = logging.LogRecord(\n name=\"test.module\",\n level=logging.INFO,\n pathname=\"/fake/path.py\",\n lineno=42,\n msg=\"Plain info message\",\n args=(),\n exc_info=None,\n )\n\n output = formatter.format(record)\n parsed = json.loads(output)\n\n assert parsed[\"level\"] == \"INFO\"\n assert parsed[\"marker\"] == \"REASON\" # default for plain messages\n assert parsed[\"intent\"] == \"Plain info message\"\n assert parsed[\"src\"] == \"test.module\"\n assert \"ts\" in parsed\n assert \"trace_id\" in parsed\n# [/DEF:test_cot_json_formatter_plain_message:Function]\n\n# [/DEF:TestLogger:Module]\n" }, { "contract_id": "TestResourceHubs", diff --git a/.opencode/skills/molecular-cot-logging/SKILL.md b/.opencode/skills/molecular-cot-logging/SKILL.md index 389a4474..9d2e7e5f 100644 --- a/.opencode/skills/molecular-cot-logging/SKILL.md +++ b/.opencode/skills/molecular-cot-logging/SKILL.md @@ -37,7 +37,7 @@ Every log record MUST be a JSON object **on a single line** with the following k | Field | Required | Type | Description | |-------|----------|------|-------------| -| `ts` | yes | string | ISO-8601 timestamp with microseconds | +| `ts` | yes | string | ISO-8601 timestamp with millisecond precision | | `level` | yes | string | Standard log level (`INFO`, `DEBUG`, `WARNING`, `ERROR`) | | `trace_id` | yes | string | UUID of the incoming HTTP request or background job | | `span_id` | no | string | UUID of the current function/block scope (optional) | @@ -45,7 +45,7 @@ Every log record MUST be a JSON object **on a single line** with the following k | `marker` | yes | string | One of `REASON`, `REFLECT`, `EXPLORE` (see below) | | `intent` | yes | string | Human-readable one-line description of what this step intends to do/verify | | `payload` | no | object | Arbitrary key-value data relevant to the step (params, result snippet) | -| `error` | no | string | Error message or reason, **only** for `EXPLORE` markers when a fallback is taken | +| `error` | conditional | string | Error message or reason. **Optional** for `REASON`/`REFLECT`, **required** for `EXPLORE` markers when a fallback or violation is taken | ### Example @@ -213,6 +213,7 @@ class TraceMiddleware(BaseHTTPMiddleware): For C4/C5 functions, a decorator that auto-emits REASON / REFLECT markers: ```python +import asyncio from functools import wraps def cot_span(marker: str = "REASON", intent: str | None = None): @@ -299,6 +300,7 @@ export function log(src, marker, intent, payload, error) { ```svelte diff --git a/.opencode/skills/semantics-testing/SKILL.md b/.opencode/skills/semantics-testing/SKILL.md index a0fa6c8b..a58513c9 100644 --- a/.opencode/skills/semantics-testing/SKILL.md +++ b/.opencode/skills/semantics-testing/SKILL.md @@ -7,6 +7,8 @@ description: Core protocol for Test Constraints, External Ontology, Graph Noise @BRIEF HOW to write tests: constraints, external ontology, graph noise reduction, and invariant traceability for pytest and vitest. @RELATION DEPENDS_ON -> [Std.Semantics.Core] @INVARIANT Test modules must trace back to production @INVARIANT tags without flooding the Semantic Graph with orphan nodes. +@RATIONALE Test contracts trace to production @INVARIANT/@POST tags via @TEST_INVARIANT, preventing orphan nodes. pytest+vitest dual stack eliminates cross-language tooling overhead. 3-edge-case floor balances coverage sufficiency against graph noise. Hardcoded fixtures block logic-mirror tautology (dominant LLM test-generation failure mode). +@REJECTED Property-based testing — non-deterministic input space creates unbounded graph edges, irreducible to fixed-scenario tracing. Snapshot testing — brittle to CSS/UI changes without invariant signal. Integration-only (no unit tests) — coarse graph edges miss localized @INVARIANT violations. Cucumber/Gherkin BDD — DSL layer breaks direct traceability to Python/Svelte @POST anchors. ## 0. QA RATIONALE (LLM PHYSICS IN TESTING) @@ -69,7 +71,7 @@ backend/tests/ ### Test module template ```python -# #region TestDashboardMigration [C:2] [TYPE Module] [SEMANTICS test,migration] +# #region TestDashboardMigration [C:3] [TYPE Module] [SEMANTICS test,migration] # @BRIEF Verify dashboard migration contracts — @POST guarantees and rejected paths. # @RELATION BINDS_TO -> [dashboard_migration] # @TEST_EDGE: missing_db_mapping -> Migration fails with MappingError diff --git a/backend/src/api/routes/admin.py b/backend/src/api/routes/admin.py index 78b10db6..aeaf5c3b 100644 --- a/backend/src/api/routes/admin.py +++ b/backend/src/api/routes/admin.py @@ -30,7 +30,10 @@ from ...schemas.auth import ( ) from ...models.auth import User, Role, ADGroupMapping from ...dependencies import has_permission, get_plugin_loader +from ...core.cot_logger import MarkerLogger from ...core.logger import logger, belief_scope + +log = MarkerLogger("AdminApi") from ...services.rbac_permission_catalog import ( discover_declared_permissions, sync_permission_catalog, @@ -167,25 +170,17 @@ async def delete_user( _=Depends(has_permission("admin:users", "WRITE")), ): with belief_scope("api.admin.delete_user"): - logger.info( - f"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}" - ) + log.reason(f"Attempting to delete user {user_id}") repo = AuthRepository(db) user = repo.get_user_by_id(user_id) if not user: - logger.warning( - f"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}" - ) + log.explore(f"User not found for deletion: {user_id}", error="User not found") raise HTTPException(status_code=404, detail="User not found") - logger.info( - f"[DEBUG] Found user to delete context={{'username': '{user.username}'}}" - ) + log.reason(f"Found user to delete: {user.username}") db.delete(user) db.commit() - logger.info( - f"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}" - ) + log.reflect(f"Successfully deleted user {user_id}") return None @@ -348,10 +343,7 @@ async def list_permissions( db=db, declared_permissions=declared_permissions ) if inserted_count > 0: - logger.info( - "[api.admin.list_permissions][Action] Synchronized %s missing RBAC permissions into auth catalog", - inserted_count, - ) + log.reason(f"Synchronized {inserted_count} missing RBAC permissions into auth catalog") repo = AuthRepository(db) return repo.list_permissions() diff --git a/backend/src/api/routes/dashboards/_listing_routes.py b/backend/src/api/routes/dashboards/_listing_routes.py index a89478e4..9bf22487 100644 --- a/backend/src/api/routes/dashboards/_listing_routes.py +++ b/backend/src/api/routes/dashboards/_listing_routes.py @@ -18,7 +18,10 @@ from src.dependencies import ( get_current_user, has_permission, ) from src.core.database import get_db +from src.core.cot_logger import MarkerLogger from src.core.logger import logger, belief_scope + +log = MarkerLogger("DashboardListing") from src.models.auth import User from src.services.profile_service import ProfileService from ._helpers import _normalize_filter_values, _dashboard_git_filter_value @@ -73,16 +76,16 @@ async def get_dashboards( f"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}", ): if page < 1: - logger.error(f"[get_dashboards][Coherence:Failed] Invalid page: {page}") + log.explore(f"Invalid page: {page}", error="Page must be >= 1") raise HTTPException(status_code=400, detail="Page must be >= 1") if page_size < 1 or page_size > 100: - logger.error(f"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}") + log.explore(f"Invalid page_size: {page_size}", error="Page size must be between 1 and 100") raise HTTPException(status_code=400, detail="Page size must be between 1 and 100") environments = config_manager.get_environments() env = next((e for e in environments if e.id == env_id), None) if not env: - logger.error(f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}") + log.explore(f"Environment not found: {env_id}", error="Environment not found") raise HTTPException(status_code=404, detail="Environment not found") bound_username: Optional[str] = None @@ -151,7 +154,8 @@ async def get_dashboards( ) except Exception as profile_error: logger.explore( - f"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}" + f"Profile preference unavailable; continuing without profile-default filter", + error=str(profile_error), ) try: @@ -193,10 +197,7 @@ async def get_dashboards( total = page_payload["total"] total_pages = page_payload["total_pages"] except Exception as page_error: - logger.warning( - "[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s", - page_error, - ) + log.explore(f"Page-based fetch failed; using compatibility fallback", error=str(page_error)) if can_apply_slug_filter: dashboards = await resource_service.get_dashboards_with_status( env, @@ -251,7 +252,7 @@ async def get_dashboards( if not actor_aliases: actor_aliases = [bound_username] logger.reason( - "[REASON] Applying profile actor filter " + f"Applying profile actor filter " f"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, " f"dashboards_before={len(dashboards)})" ) @@ -269,7 +270,7 @@ async def get_dashboards( ) if index < max_actor_samples: logger.reflect( - "[REFLECT] Profile actor filter sample " + f"Profile actor filter sample " f"(env={env_id}, dashboard_id={dashboard.get('id')}, " f"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, " f"owners={owners_value!r}, created_by={created_by_value!r}, " @@ -279,7 +280,7 @@ async def get_dashboards( filtered_dashboards.append(dashboard) logger.reflect( - "[REFLECT] Profile actor filter summary " + f"Profile actor filter summary " f"(env={env_id}, bound_username={bound_username!r}, " f"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})" ) @@ -365,10 +366,7 @@ async def get_dashboards( end_idx = start_idx + page_size paginated_dashboards = dashboards[start_idx:end_idx] - logger.info( - f"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards " - f"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})" - ) + log.reflect(f"Returning {len(paginated_dashboards)} dashboards (page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})") response_dashboards = _project_dashboard_response_items( paginated_dashboards @@ -386,9 +384,7 @@ async def get_dashboards( except HTTPException: raise except Exception as e: - logger.error( - f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}" - ) + log.explore("Failed to fetch dashboards", error=str(e)) raise HTTPException( status_code=503, detail=f"Failed to fetch dashboards: {str(e)}" ) diff --git a/backend/src/api/routes/dataset_review_pkg/_dependencies.py b/backend/src/api/routes/dataset_review_pkg/_dependencies.py index 5bb5ce13..410c59e5 100644 --- a/backend/src/api/routes/dataset_review_pkg/_dependencies.py +++ b/backend/src/api/routes/dataset_review_pkg/_dependencies.py @@ -13,8 +13,9 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, from pydantic import BaseModel, Field from sqlalchemy.orm import Session +from src.core.cot_logger import MarkerLogger from src.core.database import get_db -from src.core.logger import belief_scope, logger +from src.core.logger import belief_scope from src.dependencies import ( get_config_manager, get_current_user, @@ -69,6 +70,8 @@ from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionVersionConflictError, ) +log = MarkerLogger("DatasetReviewDeps") + # [DEF:StartSessionRequest:Class] # @COMPLEXITY: 1 @@ -445,7 +448,7 @@ def _enforce_session_version(repository, session, expected_version): try: repository.require_session_version(session, expected_version) except DatasetReviewSessionVersionConflictError as exc: - logger.explore("Dataset review optimistic-lock conflict detected", extra={"session_id": exc.session_id, "expected_version": exc.expected_version, "actual_version": exc.actual_version}) + log.explore("Dataset review optimistic-lock conflict detected", payload={"session_id": exc.session_id, "expected_version": exc.expected_version, "actual_version": exc.actual_version}, error="Optimistic lock conflict") raise _build_session_version_conflict_http_exception(exc) from exc return session diff --git a/backend/src/api/routes/dataset_review_pkg/_routes.py b/backend/src/api/routes/dataset_review_pkg/_routes.py index 4cc773e6..b1195d38 100644 --- a/backend/src/api/routes/dataset_review_pkg/_routes.py +++ b/backend/src/api/routes/dataset_review_pkg/_routes.py @@ -10,7 +10,8 @@ from typing import Any, List, Optional, Union, cast from fastapi import APIRouter, Depends, HTTPException, Query, Response, status -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.auth import User from src.models.dataset_review import ( @@ -89,6 +90,8 @@ from src.api.routes.dataset_review_pkg._dependencies import ( _build_session_version_conflict_http_exception, ) +log = MarkerLogger("DatasetReviewRoutes") + router = APIRouter(prefix="/api/dataset-orchestration", tags=["Dataset Orchestration"]) @@ -110,17 +113,17 @@ async def list_sessions( current_user: User = Depends(get_current_user), ): with belief_scope("dataset_review.list_sessions"): - logger.reason( + log.reason( "Listing dataset review sessions", - extra={"user_id": current_user.id, "page": page, "page_size": page_size}, + payload={"user_id": current_user.id, "page": page, "page_size": page_size}, ) sessions = repository.list_sessions_for_user(current_user.id) start = (page - 1) * page_size end = start + page_size items = [_serialize_session_summary(s) for s in sessions[start:end]] - logger.reflect( + log.reflect( "Session page assembled", - extra={ + payload={ "user_id": current_user.id, "returned": len(items), "total": len(sessions), @@ -156,9 +159,9 @@ async def start_session( current_user: User = Depends(get_current_user), ): with belief_scope("start_session"): - logger.reason( + log.reason( "Starting dataset review session", - extra={ + payload={ "user_id": current_user.id, "environment_id": request.environment_id, }, @@ -173,10 +176,7 @@ async def start_session( ) ) except ValueError as exc: - logger.explore( - "Session start rejected", - extra={"user_id": current_user.id, "error": str(exc)}, - ) + log.explore("Session start rejected", error="Session start rejected by orchestrator", payload={"user_id": current_user.id, "error": str(exc)}) detail = str(exc) sc = ( status.HTTP_404_NOT_FOUND @@ -184,8 +184,8 @@ async def start_session( else status.HTTP_400_BAD_REQUEST ) raise HTTPException(status_code=sc, detail=detail) from exc - logger.reflect( - "Session started", extra={"session_id": result.session.session_id} + log.reflect( + "Session started", payload={"session_id": result.session.session_id} ) return _serialize_session_summary(result.session) diff --git a/backend/src/api/routes/git/_helpers.py b/backend/src/api/routes/git/_helpers.py index 9bf90e36..ab8728d1 100644 --- a/backend/src/api/routes/git/_helpers.py +++ b/backend/src/api/routes/git/_helpers.py @@ -13,7 +13,9 @@ from typing import Optional from fastapi import HTTPException from sqlalchemy.orm import Session -from src.core.logger import logger +from src.core.cot_logger import MarkerLogger + +log = MarkerLogger("GitHelpers") from src.core.superset_client import SupersetClient from src.dependencies import get_config_manager from src.models.auth import User @@ -52,7 +54,7 @@ def _build_no_repo_status_payload() -> dict: # @PRE: `error` is a non-HTTPException instance. # @POST: Raises HTTPException(500) with route-specific context. def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None: - logger.error(f"[{route_name}][Coherence:Failed] {error}") + log.explore(f"{route_name} failed: {error}", error=str(error)) raise HTTPException(status_code=500, detail=f"{route_name} failed: {str(error)}") # [/DEF:_handle_unexpected_git_route_error:Function] @@ -66,8 +68,8 @@ def _resolve_repository_status(dashboard_id: int) -> dict: git_service = get_git_service() repo_path = git_service._get_repo_path(dashboard_id) if not os.path.exists(repo_path): - logger.debug( - f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" + log.reason( + f"Repository is not initialized for dashboard {dashboard_id}" ) return _build_no_repo_status_payload() @@ -75,8 +77,8 @@ def _resolve_repository_status(dashboard_id: int) -> dict: return git_service.get_status(dashboard_id) except HTTPException as e: if e.status_code == 404: - logger.debug( - f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" + log.reason( + f"Repository is not initialized for dashboard {dashboard_id}" ) return _build_no_repo_status_payload() raise @@ -300,11 +302,7 @@ def _resolve_current_user_git_identity( .first() ) except Exception as resolve_error: - logger.warning( - "[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s", - user_id, - resolve_error, - ) + log.explore(f"Failed to load profile preference for user {user_id}: {resolve_error}", error=str(resolve_error)) return None if not preference: diff --git a/backend/src/api/routes/git/_repo_operations_routes.py b/backend/src/api/routes/git/_repo_operations_routes.py index 6059d043..eae48840 100644 --- a/backend/src/api/routes/git/_repo_operations_routes.py +++ b/backend/src/api/routes/git/_repo_operations_routes.py @@ -10,7 +10,10 @@ from fastapi import Depends, HTTPException from sqlalchemy.orm import Session from src.core.database import get_db -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope + +log = MarkerLogger("GitRepoOps") from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.auth import User from src.models.git import GitRepository, GitServerConfig @@ -133,23 +136,13 @@ async def pull_changes( config_url = config_row.url config_provider = config_row.provider except Exception as diagnostics_error: - logger.warning( - "[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", - dashboard_id, - diagnostics_error, - ) - logger.info( - "[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s " - "binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s", - dashboard_ref, - env_id, - dashboard_id, - bool(db_repo), - (db_repo.local_path if db_repo else None), - (db_repo.remote_url if db_repo else None), - (db_repo.config_id if db_repo else None), - config_provider, - config_url, + log.explore(f"Failed to load repository binding diagnostics for dashboard {dashboard_id}: {diagnostics_error}", error=str(diagnostics_error)) + log.reason( + f"Route diagnostics dashboard_ref={dashboard_ref} env_id={env_id} resolved_dashboard_id={dashboard_id} " + f"binding_exists={bool(db_repo)} binding_local_path={(db_repo.local_path if db_repo else None)} " + f"binding_remote_url={(db_repo.remote_url if db_repo else None)} " + f"binding_config_id={(db_repo.config_id if db_repo else None)} " + f"config_provider={config_provider} config_url={config_url}" ) _apply_git_identity_from_profile(dashboard_id, db, current_user) _gs.pull_changes(dashboard_id) @@ -197,11 +190,7 @@ async def get_repository_status_batch( with belief_scope("get_repository_status_batch"): dashboard_ids = list(dict.fromkeys(request.dashboard_ids)) if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH: - logger.warning( - "[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.", - len(dashboard_ids), - MAX_REPOSITORY_STATUS_BATCH, - ) + log.explore(f"Batch size {len(dashboard_ids)} exceeds limit {MAX_REPOSITORY_STATUS_BATCH}. Truncating request.", error="Batch size exceeds limit") dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH] statuses = {} @@ -215,9 +204,7 @@ async def get_repository_status_batch( "sync_status": "ERROR", } except Exception as e: - logger.error( - f"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}" - ) + log.explore(f"Failed for dashboard {dashboard_id}: {e}", error=str(e)) statuses[str(dashboard_id)] = { **_build_no_repo_status_payload(), "sync_state": "ERROR", diff --git a/backend/src/api/routes/git/_repo_routes.py b/backend/src/api/routes/git/_repo_routes.py index 3ccb6e98..4bc3cbe1 100644 --- a/backend/src/api/routes/git/_repo_routes.py +++ b/backend/src/api/routes/git/_repo_routes.py @@ -10,7 +10,10 @@ from fastapi import Depends, HTTPException from sqlalchemy.orm import Session from src.core.database import get_db -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope + +log = MarkerLogger("GitRepoRoutes") from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.auth import User from src.models.git import GitRepository, GitServerConfig @@ -64,9 +67,7 @@ async def init_repository( raise HTTPException(status_code=404, detail="Git configuration not found") try: - logger.info( - f"[init_repository][Action] Initializing repo for dashboard {dashboard_id}" - ) + log.reason(f"Initializing repo for dashboard {dashboard_id}") _gs.init_repo( dashboard_id, init_data.remote_url, @@ -97,15 +98,10 @@ async def init_repository( db_repo.current_branch = "dev" db.commit() - logger.info( - f"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}" - ) return {"status": "success", "message": "Repository initialized"} except Exception as e: db.rollback() - logger.error( - f"[init_repository][Coherence:Failed] Failed to init repository: {e}" - ) + log.explore("Failed to init repository", error=str(e)) if isinstance(e, HTTPException): raise _handle_unexpected_git_route_error("init_repository", e) diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 3ce57760..c534e72e 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -11,7 +11,9 @@ from fastapi import APIRouter, Depends, HTTPException, status from typing import List, Optional import httpx -from ...core.logger import logger +from ...core.cot_logger import MarkerLogger + +log = MarkerLogger("LlmRoutes") from ...schemas.auth import User from ...dependencies import get_current_user as get_current_active_user from ...plugins.llm_analysis.models import ( @@ -80,8 +82,8 @@ async def get_providers( """ Get all LLM provider configurations. """ - logger.info( - f"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}" + log.reason( + f"Fetching providers for user: {current_user.username}" ) service = LLMProviderService(db) providers = service.get_all_providers() @@ -289,8 +291,8 @@ async def fetch_models_for_provider( 3. {base_url}/api/tags (Ollama format) Returns empty list if none respond — user can enter model name manually. """ - logger.info( - f"[llm_routes][fetch_models] Fetching models for {request.base_url}" + log.reason( + f"Fetching models for {request.base_url}" ) # Resolve API key: prefer direct, fall back to stored key via provider_id @@ -300,9 +302,7 @@ async def fetch_models_for_provider( svc = LLMProviderService(db) api_key = svc.get_decrypted_api_key(request.provider_id) except Exception: - logger.warning( - f"[llm_routes][fetch_models] Could not resolve API key for provider {request.provider_id}" - ) + log.explore(f"Could not resolve API key for provider {request.provider_id}", error="API key resolution failed") base = request.base_url.rstrip("/") @@ -343,8 +343,8 @@ async def fetch_models_for_provider( for m in raw ) if models: - logger.info( - f"[llm_routes][fetch_models] Found {len(models)} models at {url}" + log.reason( + f"Found {len(models)} models at {url}" ) return FetchModelsResponse( models=models, total=len(models) @@ -359,8 +359,8 @@ async def fetch_models_for_provider( for m in models_list ) if models: - logger.info( - f"[llm_routes][fetch_models] Found {len(models)} Ollama models at {url}" + log.reason( + f"Found {len(models)} Ollama models at {url}" ) return FetchModelsResponse( models=models, total=len(models) @@ -374,16 +374,11 @@ async def fetch_models_for_provider( last_error = f"Error at {url}: {str(e)[:100]}" # All endpoints failed — return empty list with no error (user can type manually) - logger.warning( - f"[llm_routes][fetch_models] No model endpoint responded for {request.base_url}. " - f"Last error: {last_error}" - ) + log.explore(f"No model endpoint responded for {request.base_url}. Last error: {last_error}", error=f"No model endpoint responded for {request.base_url}") return FetchModelsResponse(models=[], total=0) except Exception as e: - logger.error( - f"[llm_routes][fetch_models] Unexpected error for {request.base_url}: {e}" - ) + log.explore(f"Unexpected error for {request.base_url}: {e}", error=str(e)) return FetchModelsResponse(models=[], total=0) @@ -402,8 +397,8 @@ async def test_connection( current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db), ): - logger.info( - f"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}" + log.reason( + f"Testing connection for provider_id: {provider_id}" ) """ Test connection to an LLM provider. @@ -419,9 +414,7 @@ async def test_connection( # Check if API key was successfully decrypted if not api_key: - logger.error( - f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}" - ) + log.explore(f"Failed to decrypt API key for provider {provider_id}", error="API key decryption failed") raise HTTPException( status_code=500, detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.", @@ -459,8 +452,8 @@ async def test_provider_config( """ from ...plugins.llm_analysis.service import LLMClient - logger.info( - f"[llm_routes][test_provider_config][Action] Testing config for {config.name}" + log.reason( + f"Testing config for {config.name}" ) # Check if API key is provided diff --git a/backend/src/api/routes/migration.py b/backend/src/api/routes/migration.py index fd58fd75..618a5a15 100644 --- a/backend/src/api/routes/migration.py +++ b/backend/src/api/routes/migration.py @@ -346,7 +346,6 @@ async def trigger_sync_now( _=Depends(has_permission("plugin:migration", "EXECUTE")), ): with belief_scope("trigger_sync_now"): - from ...core.logger import logger from ...models.mapping import Environment as EnvironmentModel config = config_manager.get_config() @@ -366,8 +365,8 @@ async def trigger_sync_now( credentials_id=env.id, # Use env.id as credentials reference ) db.add(db_env) - logger.info( - f"[trigger_sync_now][Action] Created environment row for {env.id}" + logger.reason( + f"Created environment row for {env.id}" ) else: existing.name = env.name @@ -382,10 +381,10 @@ async def trigger_sync_now( client = SupersetClient(env) service.sync_environment(env.id, client) results["synced"].append(env.id) - logger.info(f"[trigger_sync_now][Action] Synced environment {env.id}") + logger.reason(f"Synced environment {env.id}") except Exception as e: results["failed"].append({"env_id": env.id, "error": str(e)}) - logger.error(f"[trigger_sync_now][Error] Failed to sync {env.id}: {e}") + logger.explore(f"Failed to sync {env.id}", error=str(e)) return { "status": "completed", diff --git a/backend/src/api/routes/profile.py b/backend/src/api/routes/profile.py index aab06c9f..a723f7a0 100644 --- a/backend/src/api/routes/profile.py +++ b/backend/src/api/routes/profile.py @@ -22,7 +22,8 @@ from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from ...core.database import get_db -from ...core.logger import logger, belief_scope +from ...core.logger import belief_scope +from ...core.cot_logger import MarkerLogger from ...dependencies import ( get_config_manager, get_current_user, @@ -43,6 +44,8 @@ from ...services.profile_service import ( ) # [/SECTION] +log = MarkerLogger("Profile") + router = APIRouter(prefix="/api/profile", tags=["profile"]) @@ -73,7 +76,7 @@ async def get_preferences( plugin_loader=Depends(get_plugin_loader), ): with belief_scope("profile.get_preferences", f"user_id={current_user.id}"): - logger.reason("[REASON] Resolving current user preference") + log.reason("Resolving current user preference") service = _get_profile_service(db, config_manager, plugin_loader) return service.get_my_preference(current_user) # [/DEF:get_preferences:Function] @@ -95,13 +98,13 @@ async def update_preferences( with belief_scope("profile.update_preferences", f"user_id={current_user.id}"): service = _get_profile_service(db, config_manager, plugin_loader) try: - logger.reason("[REASON] Attempting preference save") + log.reason("Attempting preference save") return service.update_my_preference(current_user=current_user, payload=payload) except ProfileValidationError as exc: - logger.reflect("[REFLECT] Preference validation failed") + log.reflect("Preference validation failed") raise HTTPException(status_code=422, detail=exc.errors) from exc except ProfileAuthorizationError as exc: - logger.explore("[EXPLORE] Cross-user mutation guard blocked request") + log.explore("Cross-user mutation guard blocked request", error="User attempted to mutate another user's resource") raise HTTPException(status_code=403, detail=str(exc)) from exc # [/DEF:update_preferences:Function] @@ -138,13 +141,13 @@ async def lookup_superset_accounts( sort_order=sort_order, ) try: - logger.reason("[REASON] Executing Superset account lookup") + log.reason("Executing Superset account lookup") return service.lookup_superset_accounts( current_user=current_user, request=lookup_request, ) except EnvironmentNotFoundError as exc: - logger.explore("[EXPLORE] Lookup request references unknown environment") + log.explore("Lookup request references unknown environment", error="EnvironmentNotFoundError raised during Superset account lookup") raise HTTPException(status_code=404, detail=str(exc)) from exc # [/DEF:lookup_superset_accounts:Function] diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index 098e3b3b..a9f8022a 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -19,7 +19,10 @@ from ...core.config_models import AppConfig, Environment, GlobalSettings, Loggin from ...models.storage import StorageConfig from ...dependencies import get_config_manager, has_permission from ...core.config_manager import ConfigManager -from ...core.logger import logger, belief_scope +from ...core.logger import belief_scope +from ...core.cot_logger import MarkerLogger + +log = MarkerLogger("Settings") from ...core.superset_client import SupersetClient from ...services.llm_prompt_templates import normalize_llm_settings from ...models.llm import ValidationPolicy @@ -98,7 +101,7 @@ async def get_settings( _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("get_settings"): - logger.info("[get_settings][Entry] Fetching all settings") + log.reason("Fetching all settings") config = config_manager.get_config().copy(deep=True) config.settings.llm = normalize_llm_settings(config.settings.llm) # Mask passwords @@ -141,7 +144,7 @@ async def update_global_settings( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("update_global_settings"): - logger.info("[update_global_settings][Entry] Updating global settings") + log.reason("Updating global settings") config_manager.update_global_settings(settings) return settings @@ -204,7 +207,7 @@ async def get_environments( _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("get_environments"): - logger.info("[get_environments][Entry] Fetching environments") + log.reason("Fetching environments") environments = config_manager.get_environments() return [ env.copy(update={"url": _normalize_superset_env_url(env.url)}) @@ -229,16 +232,14 @@ async def add_environment( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("add_environment"): - logger.info(f"[add_environment][Entry] Adding environment {env.id}") + log.reason("Adding environment", payload={"env_id": env.id}) env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) # Validate connection before adding (fast path) try: _validate_superset_connection_fast(env) except Exception as e: - logger.error( - f"[add_environment][Coherence:Failed] Connection validation failed: {e}" - ) + log.explore("Connection validation failed", payload={"env_id": env.id}, error=str(e)) raise HTTPException( status_code=400, detail=f"Connection validation failed: {e}" ) @@ -265,7 +266,7 @@ async def update_environment( config_manager: ConfigManager = Depends(get_config_manager), ): with belief_scope("update_environment"): - logger.info(f"[update_environment][Entry] Updating environment {id}") + log.reason("Updating environment", payload={"id": id}) env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) @@ -282,9 +283,7 @@ async def update_environment( try: _validate_superset_connection_fast(env_to_validate) except Exception as e: - logger.error( - f"[update_environment][Coherence:Failed] Connection validation failed: {e}" - ) + log.explore("Connection validation failed", payload={"id": id}, error=str(e)) raise HTTPException( status_code=400, detail=f"Connection validation failed: {e}" ) @@ -308,7 +307,7 @@ async def delete_environment( id: str, config_manager: ConfigManager = Depends(get_config_manager) ): with belief_scope("delete_environment"): - logger.info(f"[delete_environment][Entry] Deleting environment {id}") + log.reason("Deleting environment", payload={"id": id}) config_manager.delete_environment(id) return {"message": f"Environment {id} deleted"} @@ -328,7 +327,7 @@ async def test_environment_connection( id: str, config_manager: ConfigManager = Depends(get_config_manager) ): with belief_scope("test_environment_connection"): - logger.info(f"[test_environment_connection][Entry] Testing environment {id}") + log.reason("Testing environment connection", payload={"id": id}) # Find environment env = next((e for e in config_manager.get_environments() if e.id == id), None) @@ -338,14 +337,10 @@ async def test_environment_connection( try: _validate_superset_connection_fast(env) - logger.info( - f"[test_environment_connection][Coherence:OK] Connection successful for {id}" - ) + log.reflect("Connection successful", payload={"id": id}) return {"status": "success", "message": "Connection successful"} except Exception as e: - logger.error( - f"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}" - ) + log.explore("Connection failed", payload={"id": id}, error=str(e)) return {"status": "error", "message": str(e)} @@ -389,9 +384,7 @@ async def update_logging_config( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("update_logging_config"): - logger.info( - f"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}" - ) + log.reason("Updating logging config", payload={"level": config.level, "task_log_level": config.task_log_level}) # Get current settings and update logging config settings = config_manager.get_config().settings @@ -444,7 +437,7 @@ async def get_consolidated_settings( _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("get_consolidated_settings"): - logger.reason("Fetching consolidated settings payload") + log.reason("Fetching consolidated settings payload") config = config_manager.get_config() @@ -491,9 +484,9 @@ async def get_consolidated_settings( notifications=notifications_payload, features=config.settings.features.model_dump(), ) - logger.reflect( + log.reflect( "Consolidated settings payload assembled", - extra={ + payload={ "environment_count": len(response_payload.environments), "provider_count": len(response_payload.llm_providers), }, @@ -516,9 +509,7 @@ async def update_consolidated_settings( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("update_consolidated_settings"): - logger.info( - "[update_consolidated_settings][Entry] Applying consolidated settings patch" - ) + log.reason("Applying consolidated settings patch") current_config = config_manager.get_config() current_settings = current_config.settings diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py index 0bf574ae..6dff482e 100644 --- a/backend/src/api/routes/translate/_correction_routes.py +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User from ....dependencies import get_current_user, has_permission from ....plugins.translate.dictionary import DictionaryManager @@ -25,6 +25,7 @@ from ....schemas.translate import ( from ._router import router +log = MarkerLogger("TranslateCorrectionRoutes") # ============================================================ # Corrections @@ -44,7 +45,7 @@ async def submit_correction( db: Session = Depends(get_db), ): """Submit a term correction from a run result review.""" - logger.info(f"[translate_routes][submit_correction] User: {current_user.username}") + log.reason("Correction request", payload={"user": current_user.username}) if not payload.dictionary_id: raise HTTPException(status_code=422, detail="dictionary_id is required") try: @@ -78,7 +79,7 @@ async def submit_bulk_corrections( db: Session = Depends(get_db), ): """Submit multiple term corrections atomically.""" - logger.info(f"[translate_routes][submit_bulk_corrections] User: {current_user.username}") + log.reason("Bulk correction request", payload={"user": current_user.username}) try: corr_list = [] for c in payload.corrections: diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py index 628426a4..2c5e54a2 100644 --- a/backend/src/api/routes/translate/_dictionary_routes.py +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -13,8 +13,11 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.logger import belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User + +log = MarkerLogger("TranslateDictRoutes") from ....dependencies import get_current_user, has_permission from ....plugins.translate.dictionary import DictionaryManager from ....schemas.translate import ( @@ -49,12 +52,12 @@ async def list_dictionaries( ): """List all terminology dictionaries.""" with belief_scope("list_dictionaries"): - logger.reason(f"User: {current_user.username}") + log.reason("List dictionaries", payload={"user": current_user.username}) dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size) dict_ids = [d.id for d in dicts] counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {} items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts] - logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)}) + log.reflect("Dictionaries listed", payload={"total": total, "returned": len(items)}) return {"items": items, "total": total, "page": page, "page_size": page_size} # #endregion list_dictionaries @@ -74,12 +77,12 @@ async def get_dictionary( ): """Get a terminology dictionary by ID.""" with belief_scope("get_dictionary"): - logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + log.reason("Get dictionary", payload={"dict_id": dictionary_id, "user": current_user.username}) try: d = DictionaryManager.get_dictionary(db, dictionary_id) from ....models.translate import DictionaryEntry entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() - logger.reflect("Dictionary found", {"id": dictionary_id}) + log.reflect("Dictionary found", payload={"id": dictionary_id}) return _dict_to_response(d, entry_count) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) @@ -101,7 +104,7 @@ async def create_dictionary( ): """Create a new terminology dictionary.""" with belief_scope("create_dictionary"): - logger.reason(f"User: {current_user.username}") + log.reason("Create dictionary", payload={"user": current_user.username}) d = DictionaryManager.create_dictionary( db, name=payload.name, @@ -111,7 +114,7 @@ async def create_dictionary( description=payload.description, is_active=payload.is_active, ) - logger.reflect("Dictionary created", {"id": d.id}) + log.reflect("Dictionary created", payload={"id": d.id}) return _dict_to_response(d, 0) # #endregion create_dictionary @@ -132,7 +135,7 @@ async def update_dictionary( ): """Update a terminology dictionary.""" with belief_scope("update_dictionary"): - logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + log.reason("Update dictionary", payload={"dict_id": dictionary_id, "user": current_user.username}) try: d = DictionaryManager.update_dictionary( db, @@ -145,7 +148,7 @@ async def update_dictionary( ) from ....models.translate import DictionaryEntry entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() - logger.reflect("Dictionary updated", {"id": dictionary_id}) + log.reflect("Dictionary updated", payload={"id": dictionary_id}) return _dict_to_response(d, entry_count) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) @@ -167,10 +170,10 @@ async def delete_dictionary( ): """Delete a terminology dictionary.""" with belief_scope("delete_dictionary"): - logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + log.reason("Delete dictionary", payload={"dict_id": dictionary_id, "user": current_user.username}) try: DictionaryManager.delete_dictionary(db, dictionary_id) - logger.reflect("Dictionary deleted", {"id": dictionary_id}) + log.reflect("Dictionary deleted", payload={"id": dictionary_id}) return None except ValueError as e: if "active/scheduled" in str(e): @@ -200,7 +203,7 @@ async def list_dictionary_entries( ): """List entries for a dictionary.""" with belief_scope("list_dictionary_entries"): - logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + log.reason("List dictionary entries", payload={"dict_id": dictionary_id, "user": current_user.username}) try: entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size) items = [ @@ -216,7 +219,7 @@ async def list_dictionary_entries( } for e in entries ] - logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total}) + log.reflect("Entries listed", payload={"dict_id": dictionary_id, "total": total}) return {"items": items, "total": total, "page": page, "page_size": page_size} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) @@ -239,7 +242,7 @@ async def add_dictionary_entry( ): """Add a new entry to a dictionary.""" with belief_scope("add_dictionary_entry"): - logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + log.reason("Add dictionary entry", payload={"dict_id": dictionary_id, "user": current_user.username}) try: entry = DictionaryManager.add_entry( db, dictionary_id, @@ -247,7 +250,7 @@ async def add_dictionary_entry( target_term=payload.target_term, context_notes=payload.context_notes, ) - logger.reflect("Entry added", {"entry_id": entry.id}) + log.reflect("Entry added", payload={"entry_id": entry.id}) return { "id": entry.id, "dictionary_id": entry.dictionary_id, @@ -282,7 +285,7 @@ async def edit_dictionary_entry( ): """Update an existing dictionary entry.""" with belief_scope("edit_dictionary_entry"): - logger.reason(f"Entry: {entry_id}, User: {current_user.username}") + log.reason("Edit dictionary entry", payload={"entry_id": entry_id, "user": current_user.username}) try: entry = DictionaryManager.edit_entry( db, entry_id, @@ -290,7 +293,7 @@ async def edit_dictionary_entry( target_term=payload.target_term, context_notes=payload.context_notes, ) - logger.reflect("Entry updated", {"entry_id": entry_id}) + log.reflect("Entry updated", payload={"entry_id": entry_id}) return { "id": entry.id, "dictionary_id": entry.dictionary_id, @@ -324,10 +327,10 @@ async def delete_dictionary_entry( ): """Delete a dictionary entry.""" with belief_scope("delete_dictionary_entry"): - logger.reason(f"Entry: {entry_id}, User: {current_user.username}") + log.reason("Delete dictionary entry", payload={"entry_id": entry_id, "user": current_user.username}) try: DictionaryManager.delete_entry(db, entry_id) - logger.reflect("Entry deleted", {"entry_id": entry_id}) + log.reflect("Entry deleted", payload={"entry_id": entry_id}) return None except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) @@ -350,7 +353,7 @@ async def import_dictionary_entries( ): """Import entries into a terminology dictionary from CSV/TSV content.""" with belief_scope("import_dictionary_entries"): - logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") + log.reason("Import dictionary entries", payload={"dict_id": dictionary_id, "user": current_user.username}) if payload.on_conflict not in ("overwrite", "keep_existing", "cancel"): raise HTTPException( @@ -367,7 +370,7 @@ async def import_dictionary_entries( on_conflict=payload.on_conflict, preview_only=payload.preview_only, ) - logger.reflect("Import completed", { + log.reflect("Import completed", payload={ "dict_id": dictionary_id, "total": result["total"], "errors": len(result["errors"]), diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py index 836577fd..10617a6d 100644 --- a/backend/src/api/routes/translate/_job_routes.py +++ b/backend/src/api/routes/translate/_job_routes.py @@ -15,8 +15,11 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.logger import belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User + +log = MarkerLogger("TranslateJobRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.service import ( @@ -53,7 +56,7 @@ async def list_jobs( config_manager: ConfigManager = Depends(get_config_manager), ): """List all translation jobs.""" - logger.info(f"[translate_routes][list_jobs] User: {current_user.username}") + log.reason("Listing jobs", payload={"user": current_user.username}) try: service = TranslateJobService(db, config_manager, current_user.username) total, jobs = service.list_jobs( @@ -83,7 +86,7 @@ async def get_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Get a translation job by ID.""" - logger.info(f"[translate_routes][get_job] Job: {job_id}, User: {current_user.username}") + log.reason("Getting job", payload={"job_id": job_id, "user": current_user.username}) try: service = TranslateJobService(db, config_manager, current_user.username) job = service.get_job(job_id) @@ -109,7 +112,7 @@ async def create_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Create a new translation job.""" - logger.info(f"[translate_routes][create_job] User: {current_user.username}") + log.reason("Creating job", payload={"user": current_user.username}) try: service = TranslateJobService(db, config_manager, current_user.username) job = service.create_job(payload) @@ -136,7 +139,7 @@ async def update_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Update a translation job.""" - logger.info(f"[translate_routes][update_job] Job: {job_id}, User: {current_user.username}") + log.reason("Updating job", payload={"job_id": job_id, "user": current_user.username}) try: service = TranslateJobService(db, config_manager, current_user.username) job = service.update_job(job_id, payload) @@ -162,7 +165,7 @@ async def delete_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Delete a translation job.""" - logger.info(f"[translate_routes][delete_job] Job: {job_id}, User: {current_user.username}") + log.reason("Deleting job", payload={"job_id": job_id, "user": current_user.username}) try: service = TranslateJobService(db, config_manager, current_user.username) service.delete_job(job_id) @@ -186,7 +189,7 @@ async def duplicate_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Duplicate a translation job.""" - logger.info(f"[translate_routes][duplicate_job] Job: {job_id}, User: {current_user.username}") + log.reason("Duplicating job", payload={"job_id": job_id, "user": current_user.username}) try: service = TranslateJobService(db, config_manager, current_user.username) new_job = service.duplicate_job(job_id) @@ -223,7 +226,7 @@ async def list_datasources( datasets = service.fetch_available_datasources(env_id, search) return datasets except Exception as e: - logger.error(f"[translate_routes][list_datasources] Error: {e}") + log.explore("List datasources failed", error=str(e)) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) # #endregion list_datasources @@ -244,10 +247,11 @@ async def get_datasource_columns( config_manager: ConfigManager = Depends(get_config_manager), ): """Get column metadata and database dialect for a Superset datasource.""" - logger.info( - f"[translate_routes][get_datasource_columns] " - f"Datasource: {datasource_id}, Env: {env_id}, User: {current_user.username}" - ) + log.reason("Getting datasource columns", payload={ + "datasource_id": datasource_id, + "env_id": env_id, + "user": current_user.username, + }) try: result = fetch_datasource_columns_service( datasource_id=datasource_id, @@ -258,7 +262,7 @@ async def get_datasource_columns( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.error(f"[translate_routes][get_datasource_columns] Error: {e}") + log.explore("Get datasource columns failed", error=str(e)) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Failed to fetch datasource columns from Superset: {e}", diff --git a/backend/src/api/routes/translate/_metrics_routes.py b/backend/src/api/routes/translate/_metrics_routes.py index 4bd4fe87..87ad8ab7 100644 --- a/backend/src/api/routes/translate/_metrics_routes.py +++ b/backend/src/api/routes/translate/_metrics_routes.py @@ -10,13 +10,14 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User from ....dependencies import get_current_user, has_permission from ....plugins.translate.metrics import TranslationMetrics from ._router import router +log = MarkerLogger("TranslateMetricsRoutes") # ============================================================ # Metrics @@ -33,7 +34,7 @@ async def get_metrics( db: Session = Depends(get_db), ): """Get translation metrics.""" - logger.info(f"[translate_routes][get_metrics] User: {current_user.username}") + log.reason("Metrics request", payload={"user": current_user.username}) try: metrics_svc = TranslationMetrics(db) if job_id: @@ -55,7 +56,7 @@ async def get_job_metrics( db: Session = Depends(get_db), ): """Get aggregated metrics for a specific job.""" - logger.info(f"[translate_routes][get_job_metrics] Job: {job_id}, User: {current_user.username}") + log.reason("Job metrics request", payload={"job_id": job_id, "user": current_user.username}) try: metrics_svc = TranslationMetrics(db) return metrics_svc.get_job_metrics(job_id) diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py index e3f81ac8..4134d796 100644 --- a/backend/src/api/routes/translate/_preview_routes.py +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -14,8 +14,10 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User + +log = MarkerLogger("TranslatePreviewRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.preview import TranslationPreview @@ -50,7 +52,7 @@ async def preview_translation( config_manager: ConfigManager = Depends(get_config_manager), ): """Preview a translation before applying it.""" - logger.info(f"[translate_routes][preview_translation] Job: {job_id}, User: {current_user.username}") + log.reason("Preview translation", payload={"job_id": job_id, "user": current_user.username}) if payload is None: payload = PreviewRequest() try: @@ -65,7 +67,7 @@ async def preview_translation( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.error(f"[translate_routes][preview_translation] Error: {e}") + log.explore("Preview translation error", error=str(e)) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}") # #endregion preview_translation @@ -87,7 +89,7 @@ async def update_preview_row( config_manager: ConfigManager = Depends(get_config_manager), ): """Approve, edit, or reject a preview row.""" - logger.info(f"[translate_routes][update_preview_row] Job: {job_id}, Row: {row_key}, Action: {payload.action}") + log.reason("Update preview row", payload={"job_id": job_id, "row": row_key, "action": payload.action}) try: preview_service = TranslationPreview(db, config_manager, current_user.username) result = preview_service.update_preview_row( @@ -118,7 +120,7 @@ async def accept_preview_session( config_manager: ConfigManager = Depends(get_config_manager), ): """Accept a preview session, enabling full translation execution.""" - logger.info(f"[translate_routes][accept_preview_session] Job: {job_id}, User: {current_user.username}") + log.reason("Accept preview session", payload={"job_id": job_id, "user": current_user.username}) try: preview_service = TranslationPreview(db, config_manager, current_user.username) result = preview_service.accept_preview_session(job_id=job_id) @@ -143,7 +145,7 @@ async def apply_preview( config_manager: ConfigManager = Depends(get_config_manager), ): """Apply a preview session by session ID.""" - logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}") + log.reason("Apply preview", payload={"session_id": session_id, "user": current_user.username}) # Find job_id from session from ....models.translate import TranslationPreviewSession session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() @@ -174,7 +176,7 @@ async def get_preview_records( db: Session = Depends(get_db), ): """Get records for a preview session.""" - logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}") + log.reason("Get preview records", payload={"session_id": session_id, "user": current_user.username}) try: from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() @@ -201,7 +203,7 @@ async def get_preview_records( except HTTPException: raise except Exception as e: - logger.error(f"[translate_routes][get_preview_records] Error: {e}") + log.explore("Get preview records error", error=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_preview_records diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index 8fb6432d..8f2f5a8d 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -11,8 +11,10 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User + +log = MarkerLogger("TranslateRunListRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.orchestrator import TranslationOrchestrator @@ -43,7 +45,7 @@ async def list_runs( db: Session = Depends(get_db), ): """List all runs with cross-job filtering and pagination.""" - logger.info(f"[translate_routes][list_runs] User: {current_user.username}") + log.reason("Listing runs", payload={"user": current_user.username}) try: from ....models.translate import TranslationRun query = db.query(TranslationRun) @@ -103,7 +105,7 @@ async def list_runs( return {"items": items, "total": total, "page": page, "page_size": page_size} except Exception as e: - logger.error(f"[translate_routes][list_runs] Error: {e}") + log.explore("List runs error", error=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion list_runs @@ -126,7 +128,7 @@ async def get_run_detail( config_manager: ConfigManager = Depends(get_config_manager), ): """Get detailed run info with config_snapshot, records, events.""" - logger.info(f"[translate_routes][get_run_detail] Run: {run_id}, User: {current_user.username}") + log.reason("Get run detail", payload={"run_id": run_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) status_info = orch.get_run_status(run_id) @@ -167,7 +169,7 @@ async def download_skipped_csv( db: Session = Depends(get_db), ): """Download skipped translation records as CSV.""" - logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}") + log.reason("Download skipped CSV", payload={"run_id": run_id, "user": current_user.username}) try: from ....models.translate import TranslationRecord from fastapi.responses import StreamingResponse @@ -200,7 +202,7 @@ async def download_skipped_csv( headers={"Content-Disposition": f"attachment; filename=skipped_{run_id}.csv"}, ) except Exception as e: - logger.error(f"[translate_routes][download_skipped_csv] Error: {e}") + log.explore("Download skipped CSV error", error=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion download_skipped_csv diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index a55bd17b..f98fe312 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -14,8 +14,10 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db, SessionLocal -from ....core.logger import logger, belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User + +log = MarkerLogger("TranslateRunRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.orchestrator import TranslationOrchestrator @@ -50,7 +52,7 @@ async def run_translation( By default runs translation on preview-approved rows only. Set full_translation=true to fetch ALL rows from the Superset source dataset. """ - logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}, full={full_translation}") + log.reason("Run translation", payload={"job_id": job_id, "user": current_user.username, "full_translation": full_translation}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.start_run(job_id=job_id, is_scheduled=False) @@ -68,7 +70,7 @@ async def run_translation( if thread_run: thread_orch.execute_run(thread_run, full_translation=full_translation) except Exception as e: - logger.error(f"[translate_routes][run_translation] Background execution error: {e}") + log.explore("Background execution error", error=str(e)) finally: thread_db.close() @@ -77,7 +79,7 @@ async def run_translation( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.error(f"[translate_routes][run_translation] Error: {e}") + log.explore("Run translation error", error=str(e)) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}") # #endregion run_translation @@ -97,7 +99,7 @@ async def retry_run( config_manager: ConfigManager = Depends(get_config_manager), ): """Retry failed batches in a translation run.""" - logger.info(f"[translate_routes][retry_run] Run: {run_id}, User: {current_user.username}") + log.reason("Retry run", payload={"run_id": run_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.retry_failed_batches(run_id) @@ -105,7 +107,7 @@ async def retry_run( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.error(f"[translate_routes][retry_run] Error: {e}") + log.explore("Retry run error", error=str(e)) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}") # #endregion retry_run @@ -125,7 +127,7 @@ async def retry_insert( config_manager: ConfigManager = Depends(get_config_manager), ): """Retry the SQL insert phase for a completed run.""" - logger.info(f"[translate_routes][retry_insert] Run: {run_id}, User: {current_user.username}") + log.reason("Retry insert", payload={"run_id": run_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.retry_insert(run_id) @@ -133,7 +135,7 @@ async def retry_insert( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.error(f"[translate_routes][retry_insert] Error: {e}") + log.explore("Retry insert error", error=str(e)) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}") # #endregion retry_insert @@ -153,7 +155,7 @@ async def cancel_run( config_manager: ConfigManager = Depends(get_config_manager), ): """Cancel a running translation.""" - logger.info(f"[translate_routes][cancel_run] Run: {run_id}, User: {current_user.username}") + log.reason("Cancel run", payload={"run_id": run_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.cancel_run(run_id) @@ -177,7 +179,7 @@ async def get_run_history( config_manager: ConfigManager = Depends(get_config_manager), ): """Get run history for a translation job.""" - logger.info(f"[translate_routes][get_run_history] Job: {job_id}, User: {current_user.username}") + log.reason("Get run history", payload={"job_id": job_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) total, runs = orch.get_run_history(job_id, page=page, page_size=page_size) @@ -199,7 +201,7 @@ async def get_run_status( config_manager: ConfigManager = Depends(get_config_manager), ): """Get status and statistics for a translation run.""" - logger.info(f"[translate_routes][get_run_status] Run: {run_id}, User: {current_user.username}") + log.reason("Get run status", payload={"run_id": run_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) return orch.get_run_status(run_id) @@ -223,7 +225,7 @@ async def get_run_records( config_manager: ConfigManager = Depends(get_config_manager), ): """Get paginated records for a translation run.""" - logger.info(f"[translate_routes][get_run_records] Run: {run_id}, User: {current_user.username}") + log.reason("Get run records", payload={"run_id": run_id, "user": current_user.username}) try: orch = TranslationOrchestrator(db, config_manager, current_user.username) return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status) @@ -247,7 +249,7 @@ async def get_batches( db: Session = Depends(get_db), ): """Get batches for a translation run.""" - logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}") + log.reason("Get batches", payload={"run_id": run_id, "user": current_user.username}) try: from ....models.translate import TranslationBatch batches = ( @@ -272,7 +274,7 @@ async def get_batches( for b in batches ] except Exception as e: - logger.error(f"[translate_routes][get_batches] Error: {e}") + log.explore("Get batches error", error=str(e)) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_batches diff --git a/backend/src/api/routes/translate/_schedule_routes.py b/backend/src/api/routes/translate/_schedule_routes.py index 2916930d..adad895d 100644 --- a/backend/src/api/routes/translate/_schedule_routes.py +++ b/backend/src/api/routes/translate/_schedule_routes.py @@ -14,8 +14,10 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope +from ....core.cot_logger import MarkerLogger from ....schemas.auth import User + +log = MarkerLogger("TranslateScheduleRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.scheduler import TranslationScheduler @@ -40,7 +42,7 @@ async def get_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Get schedule for a translation job.""" - logger.info(f"[translate_routes][get_schedule] Job: {job_id}, User: {current_user.username}") + log.reason("Get schedule", payload={"job_id": job_id, "user": current_user.username}) try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.get_schedule(job_id) @@ -77,7 +79,7 @@ async def set_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Set or update schedule for a translation job.""" - logger.info(f"[translate_routes][set_schedule] Job: {job_id}, User: {current_user.username}") + log.reason("Set schedule", payload={"job_id": job_id, "user": current_user.username}) try: scheduler = TranslationScheduler(db, config_manager, current_user.username) # Check if schedule already exists @@ -139,7 +141,7 @@ async def enable_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Enable a schedule for a translation job.""" - logger.info(f"[translate_routes][enable_schedule] Job: {job_id}, User: {current_user.username}") + log.reason("Enable schedule", payload={"job_id": job_id, "user": current_user.username}) try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.set_schedule_active(job_id, is_active=True) @@ -172,7 +174,7 @@ async def disable_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Disable a schedule for a translation job.""" - logger.info(f"[translate_routes][disable_schedule] Job: {job_id}, User: {current_user.username}") + log.reason("Disable schedule", payload={"job_id": job_id, "user": current_user.username}) try: scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler.set_schedule_active(job_id, is_active=False) @@ -200,7 +202,7 @@ async def delete_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Delete schedule for a translation job.""" - logger.info(f"[translate_routes][delete_schedule] Job: {job_id}, User: {current_user.username}") + log.reason(f"Delete schedule", payload={"job_id": job_id, "user": current_user.username}) try: scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler.delete_schedule(job_id) @@ -226,7 +228,7 @@ async def get_next_executions( config_manager: ConfigManager = Depends(get_config_manager), ): """Preview next N executions for a job's schedule.""" - logger.info(f"[translate_routes][get_next_executions] Job: {job_id}, User: {current_user.username}") + log.reason(f"Get next executions", payload={"job_id": job_id, "user": current_user.username}) try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.get_schedule(job_id) diff --git a/backend/src/app.py b/backend/src/app.py index 646a10b5..a916ec53 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -29,6 +29,9 @@ from .dependencies import get_task_manager, get_scheduler_service from .core.encryption_key import ensure_encryption_key from .core.utils.network import NetworkError from .core.logger import logger, belief_scope +from .core.cot_logger import MarkerLogger + +startup_log = MarkerLogger("Startup") from .core.database import AuthSessionLocal from .core.auth.security import get_password_hash from .models.auth import User, Role @@ -136,10 +139,11 @@ def ensure_initial_admin_user() -> None: # Startup event @app.on_event("startup") async def startup_event(): - with belief_scope("startup_event"): - ensure_encryption_key() - ensure_initial_admin_user() - scheduler = get_scheduler_service() + startup_log.reason("Enter startup event") + ensure_encryption_key() + ensure_initial_admin_user() + scheduler = get_scheduler_service() + startup_log.reflect("Application startup completed") scheduler.start() @@ -163,7 +167,7 @@ async def shutdown_event(): # [/DEF:shutdown_event:Function] # [DEF:app_middleware:Block] -# @PURPOSE: Configure application-wide middleware (Session, CORS). +# @PURPOSE: Configure application-wide middleware (Session, CORS, TraceContext). # Configure Session Middleware (required by Authlib for OAuth2 flow) from .core.auth.config import auth_config @@ -177,6 +181,11 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) + +# Configure Trace Context Middleware (seeds trace_id per request) +from .core.middleware.trace import TraceContextMiddleware + +app.add_middleware(TraceContextMiddleware) # [/DEF:app_middleware:Block] @@ -187,10 +196,13 @@ app.add_middleware( # @POST: Returns 503 HTTP Exception. # @PARAM: request (Request) - The incoming request object. # @PARAM: exc (NetworkError) - The exception instance. +eh_log = MarkerLogger("ExceptionHandler") + + @app.exception_handler(NetworkError) async def network_error_handler(request: Request, exc: NetworkError): with belief_scope("network_error_handler"): - logger.error(f"Network error: {exc}") + eh_log.explore("Unhandled NetworkError in request", error=str(exc), payload={"url": str(request.url)}) return HTTPException( status_code=503, detail="Environment unavailable. Please check if the Superset instance is running.", diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py index a746ed3b..ba83089f 100644 --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -26,8 +26,10 @@ from .config_models import AppConfig, Environment, GlobalSettings from .database import SessionLocal from ..models.config import AppConfigRecord from ..models.mapping import Environment as EnvironmentRecord -from .logger import logger, configure_logger, belief_scope +from .logger import configure_logger, belief_scope +from .cot_logger import MarkerLogger +log = MarkerLogger("ConfigManager") # [DEF:ConfigManager:Class] # @COMPLEXITY: 5 @@ -45,12 +47,10 @@ class ConfigManager: def __init__(self, config_path: str = "config.json"): with belief_scope("ConfigManager.__init__"): if not isinstance(config_path, str) or not config_path: - logger.explore( - "Invalid config_path provided", extra={"path": config_path} - ) + log.explore("Invalid config_path provided", error="config_path must be a non-empty string", payload={"path": config_path}) raise ValueError("config_path must be a non-empty string") - logger.reason(f"Initializing ConfigManager with legacy path: {config_path}") + log.reason(f"Initializing ConfigManager with legacy path: {config_path}") self.config_path = Path(config_path) self.raw_payload: dict[str, Any] = {} @@ -59,13 +59,10 @@ class ConfigManager: configure_logger(self.config.settings.logging) if not isinstance(self.config, AppConfig): - logger.explore( - "Config loading resulted in invalid type", - extra={"type": type(self.config)}, - ) + log.explore("Config loading resulted in invalid type", error="Loaded config is not an AppConfig instance", payload={"type": type(self.config)}) raise TypeError("self.config must be an instance of AppConfig") - logger.reflect("ConfigManager initialization complete") + log.reflect("ConfigManager initialization complete") # [/DEF:__init__:Function] @@ -80,18 +77,18 @@ class ConfigManager: if dataset_review_env is not None: parsed = dataset_review_env.strip().lower() in ("true", "1", "yes") settings.features.dataset_review = parsed - logger.reason( + log.reason( "Applied FEATURES__DATASET_REVIEW from env", - extra={"value": parsed, "raw": dataset_review_env}, + payload={"value": parsed, "raw": dataset_review_env}, ) health_monitor_env = os.getenv("FEATURES__HEALTH_MONITOR") if health_monitor_env is not None: parsed = health_monitor_env.strip().lower() in ("true", "1", "yes") settings.features.health_monitor = parsed - logger.reason( + log.reason( "Applied FEATURES__HEALTH_MONITOR from env", - extra={"value": parsed, "raw": health_monitor_env}, + payload={"value": parsed, "raw": health_monitor_env}, ) # [/DEF:_apply_features_from_env:Function] @@ -100,7 +97,7 @@ class ConfigManager: # @PURPOSE: Build default application configuration fallback. def _default_config(self) -> AppConfig: with belief_scope("ConfigManager._default_config"): - logger.reason("Building default AppConfig fallback") + log.reason("Building default AppConfig fallback") config = AppConfig(environments=[], settings=GlobalSettings()) self._apply_features_from_env(config.settings) return config @@ -116,9 +113,9 @@ class ConfigManager: merged_payload["environments"] = typed_payload.get("environments", []) merged_payload["settings"] = typed_payload.get("settings", {}) self.raw_payload = merged_payload - logger.reason( + log.reason( "Synchronized raw payload from typed config", - extra={ + payload={ "environments_count": len( merged_payload.get("environments", []) or [] ), @@ -139,31 +136,28 @@ class ConfigManager: def _load_from_legacy_file(self) -> dict[str, Any]: with belief_scope("ConfigManager._load_from_legacy_file"): if not self.config_path.exists(): - logger.reason( + log.reason( "Legacy config file not found; using default payload", - extra={"path": str(self.config_path)}, + payload={"path": str(self.config_path)}, ) return {} - logger.reason( - "Loading legacy config file", extra={"path": str(self.config_path)} + log.reason( + "Loading legacy config file", payload={"path": str(self.config_path)} ) with self.config_path.open("r", encoding="utf-8") as fh: payload = json.load(fh) if not isinstance(payload, dict): - logger.explore( - "Legacy config payload is not a JSON object", - extra={ - "path": str(self.config_path), - "type": type(payload).__name__, - }, - ) + log.explore("Legacy config payload is not a JSON object", error=f"Expected dict, got {type(payload).__name__}", payload={ + "path": str(self.config_path), + "type": type(payload).__name__, + }) raise ValueError("Legacy config payload must be a JSON object") - logger.reason( + log.reason( "Legacy config file loaded successfully", - extra={"path": str(self.config_path), "keys": sorted(payload.keys())}, + payload={"path": str(self.config_path), "keys": sorted(payload.keys())}, ) return payload @@ -178,8 +172,8 @@ class ConfigManager: .filter(AppConfigRecord.id == "global") .first() ) - logger.reason( - "Resolved app config record", extra={"exists": record is not None} + log.reason( + "Resolved app config record", payload={"exists": record is not None} ) return record @@ -193,9 +187,9 @@ class ConfigManager: try: record = self._get_record(session) if record and isinstance(record.payload, dict): - logger.reason( + log.reason( "Loading configuration from database", - extra={"record_id": record.id}, + payload={"record_id": record.id}, ) self.raw_payload = dict(record.payload) config = AppConfig.model_validate( @@ -206,18 +200,18 @@ class ConfigManager: ) self._sync_environment_records(session, config) session.commit() - logger.reason( + log.reason( "Database configuration validated successfully", - extra={ + payload={ "environments_count": len(config.environments), "payload_keys": sorted(self.raw_payload.keys()), }, ) return config - logger.reason( + log.reason( "Database configuration record missing; attempting legacy file migration", - extra={"legacy_path": str(self.config_path)}, + payload={"legacy_path": str(self.config_path)}, ) legacy_payload = self._load_from_legacy_file() @@ -230,9 +224,9 @@ class ConfigManager: } ) self._apply_features_from_env(config.settings) - logger.reason( + log.reason( "Legacy payload validated; persisting migrated configuration to database", - extra={ + payload={ "environments_count": len(config.environments), "payload_keys": sorted(self.raw_payload.keys()), }, @@ -240,7 +234,7 @@ class ConfigManager: self._save_config_to_db(config, session=session) return config - logger.reason( + log.reason( "No persisted config found; falling back to default configuration" ) config = self._default_config() @@ -248,18 +242,12 @@ class ConfigManager: self._save_config_to_db(config, session=session) return config except (json.JSONDecodeError, TypeError, ValueError) as exc: - logger.explore( - "Recoverable config load failure; falling back to default configuration", - extra={"error": str(exc), "legacy_path": str(self.config_path)}, - ) + log.explore("Recoverable config load failure; falling back to default configuration", error=str(exc), payload={"legacy_path": str(self.config_path)}) config = self._default_config() self.raw_payload = config.model_dump() return config except Exception as exc: - logger.explore( - "Critical config load failure; re-raising persistence or validation error", - extra={"error": str(exc)}, - ) + log.explore("Critical config load failure; re-raising persistence or validation error", error=str(exc)) raise finally: session.close() @@ -291,9 +279,9 @@ class ConfigManager: record = persisted_by_id.get(normalized_id) if record is None: - logger.reason( + log.reason( "Creating relational environment record from typed config", - extra={ + payload={ "environment_id": normalized_id, "environment_name": display_name, }, @@ -327,22 +315,22 @@ class ConfigManager: payload = self._sync_raw_payload_from_config() record = self._get_record(db) if record is None: - logger.reason("Creating new global app config record") + log.reason("Creating new global app config record") record = AppConfigRecord(id="global", payload=payload) db.add(record) else: - logger.reason( + log.reason( "Updating existing global app config record", - extra={"record_id": record.id}, + payload={"record_id": record.id}, ) record.payload = payload self._sync_environment_records(db, config) db.commit() - logger.reason( + log.reason( "Configuration persisted to database", - extra={ + payload={ "environments_count": len( payload.get("environments", []) or [] ), @@ -351,7 +339,7 @@ class ConfigManager: ) except Exception: db.rollback() - logger.explore("Database save failed; transaction rolled back") + log.explore("Database save failed; transaction rolled back", error="Database transaction rolled back") raise finally: if owns_session: @@ -363,7 +351,7 @@ class ConfigManager: # @PURPOSE: Persist current in-memory configuration state. def save(self) -> None: with belief_scope("ConfigManager.save"): - logger.reason("Persisting current in-memory configuration") + log.reason("Persisting current in-memory configuration") self._save_config_to_db(self.config) # [/DEF:save:Function] @@ -389,16 +377,16 @@ class ConfigManager: def save_config(self, config: Any) -> AppConfig: with belief_scope("ConfigManager.save_config"): if isinstance(config, AppConfig): - logger.reason("Saving typed AppConfig payload") + log.reason("Saving typed AppConfig payload") self.config = config self.raw_payload = config.model_dump() self._save_config_to_db(config) return self.config if isinstance(config, dict): - logger.reason( + log.reason( "Saving raw config payload", - extra={"keys": sorted(config.keys())}, + payload={"keys": sorted(config.keys())}, ) self.raw_payload = dict(config) typed_config = AppConfig.model_validate( @@ -411,10 +399,7 @@ class ConfigManager: self._save_config_to_db(typed_config) return self.config - logger.explore( - "Unsupported config type supplied to save_config", - extra={"type": type(config).__name__}, - ) + log.explore("Unsupported config type supplied to save_config", error=f"Expected AppConfig or dict, got {type(config).__name__}", payload={"type": type(config).__name__}) raise TypeError("config must be AppConfig or dict") # [/DEF:save_config:Function] @@ -423,7 +408,7 @@ class ConfigManager: # @PURPOSE: Replace global settings and persist the resulting configuration. def update_global_settings(self, settings: GlobalSettings) -> AppConfig: with belief_scope("ConfigManager.update_global_settings"): - logger.reason("Updating global settings") + log.reason("Updating global settings") self.config.settings = settings self.save() return self.config @@ -449,12 +434,10 @@ class ConfigManager: fh.write("ok") test_file.unlink(missing_ok=True) - logger.reason("Path validation succeeded", extra={"path": str(target)}) + log.reason("Path validation succeeded", payload={"path": str(target)}) return True, "OK" except Exception as exc: - logger.explore( - "Path validation failed", extra={"path": path, "error": str(exc)} - ) + log.explore("Path validation failed", error=str(exc), payload={"path": path}) return False, str(exc) # [/DEF:validate_path:Function] @@ -507,12 +490,12 @@ class ConfigManager: item.is_default = False if existing_index is None: - logger.reason("Appending new environment", extra={"env_id": env.id}) + log.reason("Appending new environment", payload={"env_id": env.id}) self.config.environments.append(env) else: - logger.reason( + log.reason( "Replacing existing environment during add", - extra={"env_id": env.id}, + payload={"env_id": env.id}, ) self.config.environments[existing_index] = env @@ -547,13 +530,11 @@ class ConfigManager: updated.is_default = True self.config.environments[index] = updated - logger.reason("Environment updated", extra={"env_id": env_id}) + log.reason("Environment updated", payload={"env_id": env_id}) self.save() return True - logger.explore( - "Environment update skipped; env not found", extra={"env_id": env_id} - ) + log.explore("Environment update skipped; env not found", error=f"No matching environment found for id: {env_id}", payload={"env_id": env_id}) return False # [/DEF:update_environment:Function] @@ -569,10 +550,7 @@ class ConfigManager: ] if len(self.config.environments) == before: - logger.explore( - "Environment delete skipped; env not found", - extra={"env_id": env_id}, - ) + log.explore("Environment delete skipped; env not found", error=f"No matching environment found for id: {env_id}", payload={"env_id": env_id}) return False if removed and removed[0].is_default and self.config.environments: @@ -584,9 +562,9 @@ class ConfigManager: ) self.config.settings.default_environment_id = replacement - logger.reason( + log.reason( "Environment deleted", - extra={"env_id": env_id, "remaining": len(self.config.environments)}, + payload={"env_id": env_id, "remaining": len(self.config.environments)}, ) self.save() return True diff --git a/backend/src/core/cot_logger.py b/backend/src/core/cot_logger.py new file mode 100644 index 00000000..26a9e174 --- /dev/null +++ b/backend/src/core/cot_logger.py @@ -0,0 +1,258 @@ +# [DEF:CotLoggerModule:Module] +# @COMPLEXITY: 4 +# @SEMANTICS: logging, cot, molecular, structured, json, contextvar +# @PURPOSE: Structured JSON logger implementing the molecular CoT (Chain-of-Thought) logging protocol. +# Uses ContextVar for trace_id and span_id propagation across async contexts. +# Provides log(), MarkerLogger, seed_trace_id(), push_span(), pop_span(). +# @LAYER: Core +# @RELATION: [CALLED_BY] -> [TraceContextMiddleware] +# @RELATION: [CALLED_BY] -> [All C4+ service and route modules] +# @PRE: Python 3.7+ (ContextVar available). +# @POST: JSON log records written to the 'cot' logger at appropriate levels. +# @SIDE_EFFECT: Writes structured JSON to the 'cot' Python logger. +# @DATA_CONTRACT: Log call -> Single-line JSON to logging.StreamHandler/file. + +import json +import logging +import uuid +from contextvars import ContextVar +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +# [DEF:cot_trace_context:Data] +# @COMPLEXITY: 1 +# @SEMANTICS: contextvar, trace_id, span_id, propagation +# @PURPOSE: ContextVars for trace ID and span ID propagation across async boundaries. +_trace_id: ContextVar[str] = ContextVar("_trace_id", default="") +_span_id: ContextVar[str] = ContextVar("_span_id", default="") +# [/DEF:cot_trace_context:Data] + +# [DEF:cot_logger_instance:Data] +# @COMPLEXITY: 1 +# @SEMANTICS: logger, instance +# @PURPOSE: Dedicated Python logger for all CoT (molecular) log output. +cot_logger = logging.getLogger("cot") +# [/DEF:cot_logger_instance:Data] + +__all__ = [ + "seed_trace_id", + "set_trace_id", + "get_trace_id", + "push_span", + "pop_span", + "log", + "MarkerLogger", +] + +# [DEF:seed_trace_id:Function] +# @COMPLEXITY: 1 +# @SEMANTICS: trace_id, uuid, contextvar, set +# @BRIEF: Generate a new UUID4 trace_id, set it in ContextVar, and return it. +def seed_trace_id() -> str: + """Generate a new UUID4 trace ID and store it in the thread-local ContextVar. + + Returns: + str: The newly generated trace ID. + """ + trace_id = str(uuid.uuid4()) + _trace_id.set(trace_id) + return trace_id + + +# [/DEF:seed_trace_id:Function] + +# [DEF:set_trace_id:Function] +# @COMPLEXITY: 1 +# @SEMANTICS: trace_id, contextvar, set, public +# @BRIEF: Set an explicit trace_id into the ContextVar (e.g. from an incoming header). +def set_trace_id(trace_id: str) -> None: + """Set a specific trace_id into the ContextVar. + + This is used by TraceContextMiddleware when an ``X-Trace-ID`` header + is present, enabling cross-service trace chaining. + + Args: + trace_id: The trace ID to set. + """ + _trace_id.set(trace_id) + + +# [/DEF:set_trace_id:Function] + +# [DEF:get_trace_id:Function] +# @COMPLEXITY: 1 +# @SEMANTICS: trace_id, contextvar, get, public +# @BRIEF: Get the current trace_id from ContextVar. +def get_trace_id() -> str: + """Get the current trace_id from the thread-local ContextVar. + + Returns: + str: The current trace ID, or empty string if none set. + """ + return _trace_id.get() + + +# [/DEF:get_trace_id:Function] + +# [DEF:push_span:Function] +# @COMPLEXITY: 1 +# @SEMANTICS: span_id, contextvar, stack +# @BRIEF: Set a new span_id in ContextVar and return the previous span_id for later restoration. +def push_span(span: str) -> str: + """Push a new span ID onto the context and return the previous span ID. + + Args: + span: The new span identifier (e.g. function or operation name). + + Returns: + str: The previous span ID, suitable for passing to pop_span(). + """ + prev = _span_id.get() + _span_id.set(span) + return prev + + +# [/DEF:push_span:Function] + +# [DEF:pop_span:Function] +# @COMPLEXITY: 1 +# @SEMANTICS: span_id, contextvar, restore +# @BRIEF: Restore a previous span_id into the ContextVar. +def pop_span(prev: str) -> None: + """Restore a previous span ID into the ContextVar. + + Args: + prev: The span ID to restore (previously returned by push_span). + """ + _span_id.set(prev) + + +# [/DEF:pop_span:Function] + +# [DEF:cot_log_function:Function] +# @COMPLEXITY: 2 +# @SEMANTICS: log, json, structured, marker +# @BRIEF: Core structured logging function that emits a single-line JSON record. +def log( + src: str, + marker: str, + intent: str, + payload: Optional[Dict[str, Any]] = None, + error: Optional[str] = None, + level: Optional[str] = None, +) -> None: + """Emit a single-line structured JSON log record through the 'cot' logger. + + Args: + src: Qualified function or component name (e.g. 'AuthRepository.get_user'). + marker: Protocol marker — one of 'REASON', 'REFLECT', 'EXPLORE'. + intent: Short one-line description of the intent or action. + payload: Optional structured data dict to include in the record. + error: Required for EXPLORE markers; describes the violated assumption. + level: Log level override. Auto-inferred from marker if omitted + (REASON/REFLECT -> INFO, EXPLORE -> WARNING). + + Side effects: + Writes a single-line JSON string to the 'cot' Python logger. + """ + if level is None: + level = "WARNING" if marker == "EXPLORE" else "INFO" + + # Build structured extra data for the LogRecord. + # CotJsonFormatter will read these attributes to produce the final JSON output, + # consistent with the main superset_tools_app logger. + extra: Dict[str, Any] = { + "marker": marker, + "intent": intent, + "src": src, + } + + if payload is not None: + extra["payload"] = payload + + if error is not None: + extra["error"] = error + + log_func = { + "WARNING": cot_logger.warning, + "ERROR": cot_logger.error, + "DEBUG": cot_logger.debug, + }.get(level, cot_logger.info) + + log_func(intent, extra=extra) + + +# [/DEF:cot_log_function:Function] + +# [DEF:MarkerLogger:Class] +# @COMPLEXITY: 2 +# @SEMANTICS: logger, proxy, marker, syntactic-sugar +# @BRIEF: Thin proxy class providing .reason(), .reflect(), .explore() convenience methods. +class MarkerLogger: + """Thin proxy over the cot_logger.log() function. + + Usage:: + + log = MarkerLogger("AuthService") + log.reason("Validating credentials", payload={"user_id": "..."}) + log.reflect("Validation complete", payload={"result": "ok"}) + log.explore("Token expired", error="JWT expired, attempting refresh") + + Each method delegates to :func:`log` with the corresponding marker. + """ + + # [DEF:MarkerLogger.__init__:Function] + # @BRIEF: Store the module/component name used as the 'src' field in all log calls. + def __init__(self, module_name: str) -> None: + """Initialise the MarkerLogger with a module or component name. + + Args: + module_name: The value used for the 'src' field (e.g. 'AuthService'). + """ + self._module_name = module_name + + # [/DEF:MarkerLogger.__init__:Function] + + # [DEF:MarkerLogger.reason:Function] + # @BRIEF: Log a REASON marker — strict deduction, core logic. + def reason( + self, intent: str, *, payload: Optional[Dict[str, Any]] = None + ) -> None: + """Log a REASON marker (INFO level).""" + log(self._module_name, "REASON", intent, payload=payload) + + # [/DEF:MarkerLogger.reason:Function] + + # [DEF:MarkerLogger.reflect:Function] + # @BRIEF: Log a REFLECT marker — self-check, structural validation. + def reflect( + self, intent: str, *, payload: Optional[Dict[str, Any]] = None + ) -> None: + """Log a REFLECT marker (INFO level).""" + log(self._module_name, "REFLECT", intent, payload=payload) + + # [/DEF:MarkerLogger.reflect:Function] + + # [DEF:MarkerLogger.explore:Function] + # @BRIEF: Log an EXPLORE marker — searching, alternatives, violated assumptions. + def explore( + self, + intent: str, + *, + payload: Optional[Dict[str, Any]] = None, + error: Optional[str] = None, + ) -> None: + """Log an EXPLORE marker (WARNING level). + + Args: + intent: Short description of what was being attempted. + payload: Optional structured data. + error: Describes the violated assumption or error context. + """ + log(self._module_name, "EXPLORE", intent, payload=payload, error=error) + + # [/DEF:MarkerLogger.explore:Function] + + +# [/DEF:MarkerLogger:Class] +# [/DEF:CotLoggerModule:Module] diff --git a/backend/src/core/logger.py b/backend/src/core/logger.py index 5e954c7b..b07aed90 100755 --- a/backend/src/core/logger.py +++ b/backend/src/core/logger.py @@ -1,319 +1,343 @@ -# [DEF:LoggerModule:Module] -# @SEMANTICS: logging, websocket, streaming, handler -# @PURPOSE: Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets. -# @LAYER: Core -# @RELATION: Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py. -import logging -import threading -from datetime import datetime -from typing import Dict, Any, List, Optional -from collections import deque -from contextlib import contextmanager -from logging.handlers import RotatingFileHandler - -from pydantic import BaseModel, Field - -# Thread-local storage for belief state -_belief_state = threading.local() - -# Global flag for belief state logging -_enable_belief_state = True - -# Global task log level filter -_task_log_level = "INFO" - -# [DEF:BeliefFormatter:Class] -# @PURPOSE: Custom logging formatter that adds belief state prefixes to log messages. -class BeliefFormatter(logging.Formatter): - # [DEF:format:Function] - # @PURPOSE: Formats the log record, adding belief state context if available. - # @PRE: record is a logging.LogRecord. - # @POST: Returns formatted string. - # @PARAM: record (logging.LogRecord) - The log record to format. - # @RETURN: str - The formatted log message. - # @SEMANTICS: logging, formatter, context - def format(self, record): - anchor_id = getattr(_belief_state, 'anchor_id', None) - if anchor_id: - msg = str(record.msg) - # Supported molecular topology markers - markers = ("[EXPLORE]", "[REASON]", "[REFLECT]", "[COHERENCE:", "[Action]", "[Entry]", "[Exit]") - - # Avoid duplicating anchor or overriding explicit markers - if msg.startswith(f"[{anchor_id}]"): - pass - elif any(msg.startswith(m) for m in markers): - record.msg = f"[{anchor_id}]{msg}" - else: - # Default covalent bond - record.msg = f"[{anchor_id}][Action] {msg}" - - return super().format(record) - # [/DEF:format:Function] -# [/DEF:BeliefFormatter:Class] - -# Re-using LogEntry from task_manager for consistency -# [DEF:LogEntry:Class] -# @SEMANTICS: log, entry, record, pydantic -# @PURPOSE: A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py. -class LogEntry(BaseModel): - timestamp: datetime = Field(default_factory=datetime.utcnow) - level: str - message: str - context: Optional[Dict[str, Any]] = None - -# [/DEF:LogEntry:Class] - -# [DEF:belief_scope:Function] -# @PURPOSE: Context manager for structured Belief State logging. -# @PARAM: anchor_id (str) - The identifier for the current semantic block. -# @PARAM: message (str) - Optional entry message. -# @PRE: anchor_id must be provided. -# @POST: Thread-local belief state is updated and entry/exit logs are generated. -# @SEMANTICS: logging, context, belief_state -@contextmanager -def belief_scope(anchor_id: str, message: str = ""): - # Log Entry if enabled (DEBUG level to reduce noise) - if _enable_belief_state: - entry_msg = f"[{anchor_id}][Entry]" - if message: - entry_msg += f" {message}" - logger.debug(entry_msg) - - # Set thread-local anchor_id - old_anchor = getattr(_belief_state, 'anchor_id', None) - _belief_state.anchor_id = anchor_id - - try: - yield - # Log Coherence OK and Exit (DEBUG level to reduce noise) - logger.debug("[COHERENCE:OK]") - if _enable_belief_state: - logger.debug("[Exit]") - except Exception as e: - # Log Coherence Failed (DEBUG level to reduce noise) - logger.debug(f"[COHERENCE:FAILED] {str(e)}") - raise - finally: - # Restore old anchor - _belief_state.anchor_id = old_anchor - -# [/DEF:belief_scope:Function] - -# [DEF:configure_logger:Function] -# @PURPOSE: Configures the logger with the provided logging settings. -# @PRE: config is a valid LoggingConfig instance. -# @POST: Logger level, handlers, belief state flag, and task log level are updated. -# @PARAM: config (LoggingConfig) - The logging configuration. -# @SEMANTICS: logging, configuration, initialization -def configure_logger(config): - global _enable_belief_state, _task_log_level - _enable_belief_state = config.enable_belief_state - _task_log_level = config.task_log_level.upper() - - # Set logger level - level = getattr(logging, config.level.upper(), logging.INFO) - logger.setLevel(level) - - # Remove existing file handlers - handlers_to_remove = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)] - for h in handlers_to_remove: - logger.removeHandler(h) - h.close() - - # Add file handler if file_path is set - if config.file_path: - from pathlib import Path - log_file = Path(config.file_path) - log_file.parent.mkdir(parents=True, exist_ok=True) - - file_handler = RotatingFileHandler( - config.file_path, - maxBytes=config.max_bytes, - backupCount=config.backup_count - ) - file_handler.setFormatter(BeliefFormatter( - '[%(asctime)s][%(levelname)s][%(name)s] %(message)s' - )) - logger.addHandler(file_handler) - - # Update existing handlers' formatters to BeliefFormatter - for handler in logger.handlers: - if not isinstance(handler, RotatingFileHandler): - handler.setFormatter(BeliefFormatter( - '[%(asctime)s][%(levelname)s][%(name)s] %(message)s' - )) -# [/DEF:configure_logger:Function] - -# [DEF:get_task_log_level:Function] -# @PURPOSE: Returns the current task log level filter. -# @PRE: None. -# @POST: Returns the task log level string. -# @RETURN: str - The current task log level (DEBUG, INFO, WARNING, ERROR). -# @SEMANTICS: logging, configuration, getter -def get_task_log_level() -> str: - """Returns the current task log level filter.""" - return _task_log_level -# [/DEF:get_task_log_level:Function] - -# [DEF:should_log_task_level:Function] -# @PURPOSE: Checks if a log level should be recorded based on task_log_level setting. -# @PRE: level is a valid log level string. -# @POST: Returns True if level meets or exceeds task_log_level threshold. -# @PARAM: level (str) - The log level to check. -# @RETURN: bool - True if the level should be logged. -# @SEMANTICS: logging, filter, level -def should_log_task_level(level: str) -> bool: - """Checks if a log level should be recorded based on task_log_level setting.""" - level_order = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3} - current_level = _task_log_level.upper() - check_level = level.upper() - - current_order = level_order.get(current_level, 1) # Default to INFO - check_order = level_order.get(check_level, 1) - - return check_order >= current_order -# [/DEF:should_log_task_level:Function] - -# [DEF:WebSocketLogHandler:Class] -# @SEMANTICS: logging, handler, websocket, buffer -# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets. -class WebSocketLogHandler(logging.Handler): - """ - A logging handler that stores log records and can be extended to send them - over WebSockets. - """ - # [DEF:__init__:Function] - # @PURPOSE: Initializes the handler with a fixed-capacity buffer. - # @PRE: capacity is an integer. - # @POST: Instance initialized with empty deque. - # @PARAM: capacity (int) - Maximum number of logs to keep in memory. - # @SEMANTICS: logging, initialization, buffer - def __init__(self, capacity: int = 1000): - super().__init__() - self.log_buffer: deque[LogEntry] = deque(maxlen=capacity) - # In a real implementation, you'd have a way to manage active WebSocket connections - # e.g., self.active_connections: Set[WebSocket] = set() - # [/DEF:__init__:Function] - - # [DEF:emit:Function] - # @PURPOSE: Captures a log record, formats it, and stores it in the buffer. - # @PRE: record is a logging.LogRecord. - # @POST: Log is added to the log_buffer. - # @PARAM: record (logging.LogRecord) - The log record to emit. - # @SEMANTICS: logging, handler, buffer - def emit(self, record: logging.LogRecord): - try: - log_entry = LogEntry( - level=record.levelname, - message=self.format(record), - context={ - "name": record.name, - "pathname": record.pathname, - "lineno": record.lineno, - "funcName": record.funcName, - "process": record.process, - "thread": record.thread, - } - ) - self.log_buffer.append(log_entry) - # Here you would typically send the log_entry to all active WebSocket connections - # for real-time streaming to the frontend. - # Example: for ws in self.active_connections: await ws.send_json(log_entry.dict()) - except Exception: - self.handleError(record) - # [/DEF:emit:Function] - - # [DEF:get_recent_logs:Function] - # @PURPOSE: Returns a list of recent log entries from the buffer. - # @PRE: None. - # @POST: Returns list of LogEntry objects. - # @RETURN: List[LogEntry] - List of buffered log entries. - # @SEMANTICS: logging, buffer, retrieval - def get_recent_logs(self) -> List[LogEntry]: - """ - Returns a list of recent log entries from the buffer. - """ - return list(self.log_buffer) - # [/DEF:get_recent_logs:Function] - -# [/DEF:WebSocketLogHandler:Class] - -# [DEF:Logger:Global] -# @SEMANTICS: logger, global, instance -# @PURPOSE: The global logger instance for the application, configured with both a console handler and the custom WebSocket handler. -logger = logging.getLogger("superset_tools_app") - -# [DEF:believed:Function] -# @PURPOSE: A decorator that wraps a function in a belief scope. -# @PARAM: anchor_id (str) - The identifier for the semantic block. -# @PRE: anchor_id must be a string. -# @POST: Returns a decorator function. -def believed(anchor_id: str): - # [DEF:decorator:Function] - # @PURPOSE: Internal decorator for belief scope. - # @PRE: func must be a callable. - # @POST: Returns the wrapped function. - def decorator(func): - # [DEF:wrapper:Function] - # @PURPOSE: Internal wrapper that enters belief scope. - # @PRE: None. - # @POST: Executes the function within a belief scope. - def wrapper(*args, **kwargs): - with belief_scope(anchor_id): - return func(*args, **kwargs) - # [/DEF:wrapper:Function] - return wrapper - # [/DEF:decorator:Function] - return decorator -# [/DEF:believed:Function] -logger.setLevel(logging.INFO) - -# Create a formatter -formatter = BeliefFormatter( - '[%(asctime)s][%(levelname)s][%(name)s] %(message)s' -) - -# Add console handler -console_handler = logging.StreamHandler() -console_handler.setFormatter(formatter) -logger.addHandler(console_handler) - -# Add WebSocket log handler -websocket_log_handler = WebSocketLogHandler() -websocket_log_handler.setFormatter(formatter) -logger.addHandler(websocket_log_handler) - -# Example usage: -# logger.info("Application started", extra={"context_key": "context_value"}) -# logger.error("An error occurred", exc_info=True) - -import types - -# [DEF:explore:Function] -# @PURPOSE: Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses. -# @SEMANTICS: log, explore, molecule -def explore(self, msg, *args, **kwargs): - self.warning(f"[EXPLORE] {msg}", *args, **kwargs) -# [/DEF:explore:Function] - -# [DEF:reason:Function] -# @PURPOSE: Logs a REASON message (Covalent bond) for strict deduction and core logic. -# @SEMANTICS: log, reason, molecule -def reason(self, msg, *args, **kwargs): - self.info(f"[REASON] {msg}", *args, **kwargs) -# [/DEF:reason:Function] - -# [DEF:reflect:Function] -# @PURPOSE: Logs a REFLECT message (Hydrogen bond) for self-check and structural validation. -# @SEMANTICS: log, reflect, molecule -def reflect(self, msg, *args, **kwargs): - self.debug(f"[REFLECT] {msg}", *args, **kwargs) -# [/DEF:reflect:Function] - -logger.explore = types.MethodType(explore, logger) -logger.reason = types.MethodType(reason, logger) -logger.reflect = types.MethodType(reflect, logger) - -# [/DEF:Logger:Global] -# [/DEF:LoggerModule:Module] \ No newline at end of file +# #region LoggerModule [C:5] [TYPE Module] [SEMANTICS logging,websocket,streaming,handler,cot,json] +# @BRIEF Application logging system with CotJsonFormatter producing molecular CoT JSON output. +# @LAYER Core +# @RELATION USED_BY -> [All application modules] +# @RELATION DEPENDS_ON -> [CotLoggerModule] +# @RELATION DEPENDS_ON -> [WebSocketLogHandler] +# @PRE Python 3.7+ with cot_logger ContextVars available. +# @POST All log output is single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id. +# @SIDE_EFFECT Configures global logger handlers and formatters; seeds global _enable_belief_state and _task_log_level; writes JSON to console/files. +# @RATIONALE Replaced BeliefFormatter text-based [Entry]/[Exit]/[Action] logging with CotJsonFormatter JSON output. Old format was not machine-parseable and mixed presentation with intent. CotJsonFormatter produces structured JSON consistent with cot_logger.py MarkerLogger protocol. +# @REJECTED Keeping both formatters side-by-side was rejected (two inconsistent output formats would confuse log consumers). +# @DATA_CONTRACT All log records conform to molecular CoT JSON schema: {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?} +# @INVARIANT CotJsonFormatter.format() always returns valid single-line JSON string. +import json +import logging +import threading +import types +from datetime import datetime, timezone +from contextlib import contextmanager +from logging.handlers import RotatingFileHandler + +from .cot_logger import _trace_id, _span_id +from .ws_log_handler import WebSocketLogHandler, LogEntry # noqa: F401 + +# Thread-local storage for belief state +_belief_state = threading.local() + +# Global flag for belief state logging +_enable_belief_state = True + +# Global task log level filter +_task_log_level = "INFO" + + +# #region CotJsonFormatter [C:3] [TYPE Class] [SEMANTICS logging,formatter,json,cot,protocol] +# @BRIEF JSON formatter implementing the molecular CoT logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id. +# @RELATION IMPLEMENTS -> [CotJsonFormat] +class CotJsonFormatter(logging.Formatter): + """JSON formatter implementing the molecular CoT logging protocol. + + Reads structured data from the LogRecord's extra attributes (marker, intent, payload, + error, src) set via the ``extra`` kwarg. Falls back to plain message wrapping when no + structured extra is present. + + Output format (single-line JSON):: + + { + "ts": "2026-05-12T10:30:00.123", + "level": "INFO", + "trace_id": "uuid-or-no-trace", + "src": "module.name", + "marker": "REASON|REFLECT|EXPLORE", + "intent": "human-readable intent", + "span_id": "optional-span", + "payload": { ... }, # optional + "error": "..." # optional + } + """ + + # #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record] + # @BRIEF Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields. + def format(self, record): + # Try to extract structured data from extra kwargs on the record + marker = getattr(record, 'marker', None) + intent = getattr(record, 'intent', None) + payload = getattr(record, 'payload', None) + error = getattr(record, 'error', None) + src = getattr(record, 'src', None) or record.name + + # For records that already come as JSON strings (e.g. from cot_logger.log()), + # detect and pass through the message directly as intent. + if not marker: + marker = "REASON" # default for plain messages + if not intent: + intent = record.getMessage() + + log_obj = { + "ts": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "level": record.levelname, + "trace_id": _trace_id.get() or "no-trace", + "src": src, + "marker": marker, + "intent": intent, + } + + span_id = _span_id.get() + if span_id: + log_obj["span_id"] = span_id + if payload: + log_obj["payload"] = payload + if error: + log_obj["error"] = error + + return json.dumps(log_obj, ensure_ascii=False, default=str) + # #endregion CotJsonFormatter.format + + +# #endregion CotJsonFormatter + + +# #region belief_scope [C:4] [TYPE Function] [SEMANTICS logging,context,belief_state,cot,marker] +# @BRIEF Context manager for structured Belief State logging with CoT markers (REASON on entry, REFLECT on success, EXPLORE on failure). +# @RELATION CALLED_BY -> [All C3+ modules using belief_scope] +# @RELATION BINDS_TO -> [CotJsonFormatter] +# @PRE anchor_id must be provided. +# @POST Thread-local belief state is updated. Entry logged as REASON marker, success coherence as REFLECT marker, failure as EXPLORE marker. +# @SIDE_EFFECT Writes debug-level log records with structured extra data; mutates _belief_state.anchor_id. +@contextmanager +def belief_scope(anchor_id: str, message: str = ""): + """Context manager for structured Belief State logging with CoT markers. + + Emits structured markers via the ``extra`` dict so that CotJsonFormatter + produces proper JSON output. + + Usage:: + + with belief_scope("MyFunction", "Processing request"): + logger.info("Doing work") + """ + old_anchor = getattr(_belief_state, 'anchor_id', None) + _belief_state.anchor_id = anchor_id + + # Log Entry (REASON marker) — only when belief state logging is enabled + if _enable_belief_state: + entry_msg = message or f"Enter {anchor_id}" + logger.debug(entry_msg, extra={ + "marker": "REASON", + "intent": entry_msg, + "src": anchor_id, + }) + + try: + yield + # Coherence OK / scope exit — always logged for internal tracking + logger.debug("Coherence OK", extra={ + "marker": "REFLECT", + "intent": "Coherence OK", + "src": anchor_id, + "payload": {"contract_id": anchor_id}, + }) + except Exception as e: + # Coherence FAILED — always logged for error tracking + logger.debug(f"Coherence FAILED: {e}", extra={ + "marker": "EXPLORE", + "intent": "Coherence FAILED", + "error": str(e), + "src": anchor_id, + "payload": {"contract_id": anchor_id}, + }) + raise + finally: + # Restore old anchor + _belief_state.anchor_id = old_anchor + +# #endregion belief_scope + + +# #region configure_logger [C:4] [TYPE Function] [SEMANTICS logging,configuration,initialization,cot] +# @BRIEF Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags. +# @RELATION CALLS -> [CotJsonFormatter] +# @RELATION CALLS -> [CotLoggerModule] +# @PRE config is a valid LoggingConfig instance. +# @POST Logger levels, handlers, formatters, belief state flag, and task log level are updated. The 'cot' logger has CotJsonFormatter with propagation disabled. +# @SIDE_EFFECT Mutates global _enable_belief_state, _task_log_level; adds/removes handlers on both logger and cot_logger; creates file directories. +def configure_logger(config): + global _enable_belief_state, _task_log_level + _enable_belief_state = config.enable_belief_state + _task_log_level = config.task_log_level.upper() + + # Set logger level + level = getattr(logging, config.level.upper(), logging.INFO) + logger.setLevel(level) + + # Remove existing file handlers + handlers_to_remove = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)] + for h in handlers_to_remove: + logger.removeHandler(h) + h.close() + + # Add file handler if file_path is set + if config.file_path: + from pathlib import Path + log_file = Path(config.file_path) + log_file.parent.mkdir(parents=True, exist_ok=True) + + file_handler = RotatingFileHandler( + config.file_path, + maxBytes=config.max_bytes, + backupCount=config.backup_count + ) + file_handler.setFormatter(CotJsonFormatter()) + logger.addHandler(file_handler) + + # Update existing handlers' formatters to CotJsonFormatter + for handler in logger.handlers: + if not isinstance(handler, RotatingFileHandler): + handler.setFormatter(CotJsonFormatter()) + + # Also configure the 'cot' logger for consistent JSON output + from .cot_logger import cot_logger + cot_logger.setLevel(level) + # Remove all existing handlers from the cot logger + for h in list(cot_logger.handlers): + cot_logger.removeHandler(h) + h.close() + # Add a console handler with CotJsonFormatter (if file path set, add file handler too) + cot_console = logging.StreamHandler() + cot_console.setFormatter(CotJsonFormatter()) + cot_logger.addHandler(cot_console) + if config.file_path: + from pathlib import Path + log_file = Path(config.file_path) + log_file.parent.mkdir(parents=True, exist_ok=True) + cot_file = RotatingFileHandler( + config.file_path, + maxBytes=config.max_bytes, + backupCount=config.backup_count + ) + cot_file.setFormatter(CotJsonFormatter()) + cot_logger.addHandler(cot_file) + # Disable propagation so cot messages don't double-emit via root loggers + cot_logger.propagate = False + +# #endregion configure_logger + + +# #region get_task_log_level [C:1] [TYPE Function] [SEMANTICS logging,configuration,getter] +def get_task_log_level() -> str: + """Returns the current task log level filter.""" + return _task_log_level + +# #endregion get_task_log_level + + +# #region should_log_task_level [C:2] [TYPE Function] [SEMANTICS logging,filter,level] +# @BRIEF Checks if a log level should be recorded based on the current task log level threshold. +def should_log_task_level(level: str) -> bool: + """Checks if a log level should be recorded based on task_log_level setting.""" + level_order = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3} + current_level = _task_log_level.upper() + check_level = level.upper() + + current_order = level_order.get(current_level, 1) # Default to INFO + check_order = level_order.get(check_level, 1) + + return check_order >= current_order + +# #endregion should_log_task_level + + +# #region Logger [C:2] [TYPE Global] [SEMANTICS logger,global,instance] +# @BRIEF The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler. +logger = logging.getLogger("superset_tools_app") +logger.setLevel(logging.INFO) + +# Create CotJsonFormatter +_formatter = CotJsonFormatter() + +# Add console handler +console_handler = logging.StreamHandler() +console_handler.setFormatter(_formatter) +logger.addHandler(console_handler) + +# Add WebSocket log handler +websocket_log_handler = WebSocketLogHandler() +websocket_log_handler.setFormatter(_formatter) +logger.addHandler(websocket_log_handler) + +# Example usage: +# logger.info("Application started", extra={"context_key": "context_value"}) +# logger.error("An error occurred", exc_info=True) + + +# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning] +# @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter. +def explore(self, msg, *args, **kwargs): + """Log an EXPLORE marker — searching, alternatives, violated assumptions. + + Passes structured ``extra`` data so CotJsonFormatter produces proper JSON. + """ + user_extra = kwargs.pop('extra', {}) + extra = {'marker': 'EXPLORE', 'intent': msg} + extra.update(user_extra) + self.warning(msg, *args, extra=extra, **kwargs) + +# #endregion explore + + +# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info] +# @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter. +def reason(self, msg, *args, **kwargs): + """Log a REASON marker — strict deduction, core logic. + + Passes structured ``extra`` data so CotJsonFormatter produces proper JSON. + """ + user_extra = kwargs.pop('extra', {}) + extra = {'marker': 'REASON', 'intent': msg} + extra.update(user_extra) + self.info(msg, *args, extra=extra, **kwargs) + +# #endregion reason + + +# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,debug] +# @BRIEF Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter. +def reflect(self, msg, *args, **kwargs): + """Log a REFLECT marker — self-check, structural validation. + + Passes structured ``extra`` data so CotJsonFormatter produces proper JSON. + """ + user_extra = kwargs.pop('extra', {}) + extra = {'marker': 'REFLECT', 'intent': msg} + extra.update(user_extra) + self.debug(msg, *args, extra=extra, **kwargs) + +# #endregion reflect + + +logger.explore = types.MethodType(explore, logger) +logger.reason = types.MethodType(reason, logger) +logger.reflect = types.MethodType(reflect, logger) + +# #endregion Logger + + +# #region believed [C:2] [TYPE Function] [SEMANTICS decorator,belief_scope,wrapper] +# @BRIEF Decorator that wraps a function in a belief_scope context manager. +def believed(anchor_id: str): + # #region believed.decorator [C:1] [TYPE Function] [SEMANTICS decorator,inner] + def decorator(func): + # #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope] + def wrapper(*args, **kwargs): + with belief_scope(anchor_id): + return func(*args, **kwargs) + # #endregion believed.decorator.wrapper + return wrapper + # #endregion believed.decorator + return decorator + +# #endregion believed + + +# #endregion LoggerModule diff --git a/backend/src/core/logger/__tests__/test_logger.py b/backend/src/core/logger/__tests__/test_logger.py index c93f9028..974e1724 100644 --- a/backend/src/core/logger/__tests__/test_logger.py +++ b/backend/src/core/logger/__tests__/test_logger.py @@ -1,6 +1,6 @@ # [DEF:test_logger:Module] # @COMPLEXITY: 3 -# @PURPOSE: Unit tests for logger module +# @PURPOSE: Unit tests for logger module — CoT JSON format markers. # @LAYER: Infra # @RELATION: VERIFIES -> src.core.logger @@ -12,12 +12,14 @@ sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src")) import pytest import logging +import json from src.core.logger import ( belief_scope, logger, configure_logger, get_task_log_level, - should_log_task_level + should_log_task_level, + CotJsonFormatter, ) from src.core.config_models import LoggingConfig @@ -43,13 +45,13 @@ def reset_logger_state(): configure_logger(config) -# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function] +# [DEF:test_belief_scope_logs_reason_reflect_at_debug:Function] # @RELATION: BINDS_TO -> test_logger -# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level. +# @PURPOSE: Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level. # @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level. -def test_belief_scope_logs_entry_action_exit_at_debug(caplog): - """Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.""" +# @POST: Logs are verified to contain REASON (entry) and REFLECT (coherence) markers. +def test_belief_scope_logs_reason_reflect_at_debug(caplog): + """Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -57,32 +59,43 @@ def test_belief_scope_logs_entry_action_exit_at_debug(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("TestFunction"): logger.info("Doing something important") - # Check that the logs contain the expected patterns - log_messages = [record.message for record in caplog.records] + # Check that the records contain the expected markers + reason_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction' + ] + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction' + ] + info_records = [ + r for r in caplog.records + if r.levelname == 'INFO' and 'Doing something important' in r.getMessage() + ] + + assert len(reason_records) >= 1, "REASON marker not found for TestFunction" + assert len(reflect_records) >= 1, "REFLECT marker not found for TestFunction" + assert len(info_records) >= 1, "INFO log 'Doing something important' not found" - assert any("[TestFunction][Entry]" in msg for msg in log_messages), "Entry log not found" - assert any("[TestFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found" - assert any("[TestFunction][Exit]" in msg for msg in log_messages), "Exit log not found" - # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) -# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function] +# [/DEF:test_belief_scope_logs_reason_reflect_at_debug:Function] # [DEF:test_belief_scope_error_handling:Function] # @RELATION: BINDS_TO -> test_logger -# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception. +# @PURPOSE: Test that belief_scope logs EXPLORE marker on exception. # @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST: Logs are verified to contain Coherence:Failed tag. +# @POST: Logs are verified to contain EXPLORE marker with error info. def test_belief_scope_error_handling(caplog): - """Test that belief_scope logs Coherence:Failed on exception.""" + """Test that belief_scope logs EXPLORE marker on exception.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -90,19 +103,24 @@ def test_belief_scope_error_handling(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with pytest.raises(ValueError): with belief_scope("FailingFunction"): raise ValueError("Something went wrong") - log_messages = [record.message for record in caplog.records] + # Check that an EXPLORE marker was emitted + explore_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'EXPLORE' + and getattr(r, 'src', None) == 'FailingFunction' + ] + + assert len(explore_records) >= 1, "EXPLORE marker not found for FailingFunction" + assert 'Something went wrong' in explore_records[0].getMessage() or \ + 'Something went wrong' in str(getattr(explore_records[0], 'error', '')) - assert any("[FailingFunction][Entry]" in msg for msg in log_messages), "Entry log not found" - assert any("[FailingFunction][COHERENCE:FAILED]" in msg for msg in log_messages), "Failed coherence log not found" - # Exit should not be logged on failure - # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) @@ -111,11 +129,11 @@ def test_belief_scope_error_handling(caplog): # [DEF:test_belief_scope_success_coherence:Function] # @RELATION: BINDS_TO -> test_logger -# @PURPOSE: Test that belief_scope logs Coherence:OK on success. +# @PURPOSE: Test that belief_scope logs REFLECT marker on success. # @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST: Logs are verified to contain Coherence:OK tag. +# @POST: Logs are verified to contain REFLECT marker. def test_belief_scope_success_coherence(caplog): - """Test that belief_scope logs Coherence:OK on success.""" + """Test that belief_scope logs REFLECT marker on success.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -123,41 +141,58 @@ def test_belief_scope_success_coherence(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("SuccessFunction"): pass - log_messages = [record.message for record in caplog.records] + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' + and getattr(r, 'src', None) == 'SuccessFunction' + ] + + assert len(reflect_records) >= 1, "REFLECT marker not found for SuccessFunction" + assert 'Coherence OK' in reflect_records[0].getMessage() or \ + getattr(reflect_records[0], 'intent', '') == 'Coherence OK' - assert any("[SuccessFunction][COHERENCE:OK]" in msg for msg in log_messages), "Success coherence log not found" - # [/DEF:test_belief_scope_success_coherence:Function] -# [DEF:test_belief_scope_not_visible_at_info:Function] +# [DEF:test_belief_scope_reason_not_visible_at_info:Function] # @RELATION: BINDS_TO -> test_logger -# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level. +# @PURPOSE: Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level. # @PRE: belief_scope is available. caplog fixture is used. -# @POST: Entry/Exit/Coherence logs are not captured at INFO level. -def test_belief_scope_not_visible_at_info(caplog): - """Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.""" +# @POST: REASON/REFLECT markers are not captured at INFO level. +def test_belief_scope_reason_not_visible_at_info(caplog): + """Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.""" caplog.set_level("INFO") with belief_scope("InfoLevelFunction"): logger.info("Doing something important") - log_messages = [record.message for record in caplog.records] + # The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO + reason_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction' + ] + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction' + ] - # Action log should be visible - assert any("[InfoLevelFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found" - # Entry/Exit/Coherence should NOT be visible at INFO level - assert not any("[InfoLevelFunction][Entry]" in msg for msg in log_messages), "Entry log should not be visible at INFO" - assert not any("[InfoLevelFunction][Exit]" in msg for msg in log_messages), "Exit log should not be visible at INFO" - assert not any("[InfoLevelFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence log should not be visible at INFO" -# [/DEF:test_belief_scope_not_visible_at_info:Function] + assert len(reason_records) == 0, "REASON marker should not be visible at INFO" + assert len(reflect_records) == 0, "REFLECT marker should not be visible at INFO" + + # But the INFO-level message should be visible + info_records = [ + r for r in caplog.records + if r.levelname == 'INFO' and 'Doing something important' in r.getMessage() + ] + assert len(info_records) >= 1, "INFO log 'Doing something important' should be visible" +# [/DEF:test_belief_scope_reason_not_visible_at_info:Function] # [DEF:test_task_log_level_default:Function] @@ -200,7 +235,7 @@ def test_configure_logger_task_log_level(): enable_belief_state=True ) configure_logger(config) - + assert get_task_log_level() == "DEBUG", "task_log_level should be DEBUG" assert should_log_task_level("DEBUG") is True, "DEBUG should be logged at DEBUG threshold" # [/DEF:test_configure_logger_task_log_level:Function] @@ -208,11 +243,11 @@ def test_configure_logger_task_log_level(): # [DEF:test_enable_belief_state_flag:Function] # @RELATION: BINDS_TO -> test_logger -# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging. +# @PURPOSE: Test that enable_belief_state flag controls belief_scope entry logging. # @PRE: LoggingConfig is available. caplog fixture is used. -# @POST: belief_scope logs are controlled by the flag. +# @POST: REASON entry marker is suppressed when disabled; REFLECT coherence still logged. def test_enable_belief_state_flag(caplog): - """Test that enable_belief_state flag controls belief_scope logging.""" + """Test that enable_belief_state flag controls belief_scope REASON entry logging.""" # Disable belief state config = LoggingConfig( level="DEBUG", @@ -220,19 +255,25 @@ def test_enable_belief_state_flag(caplog): enable_belief_state=False ) configure_logger(config) - + caplog.set_level("DEBUG") - + with belief_scope("DisabledFunction"): logger.info("Doing something") - - log_messages = [record.message for record in caplog.records] - - # Entry and Exit should NOT be logged when disabled - assert not any("[DisabledFunction][Entry]" in msg for msg in log_messages), "Entry should not be logged when disabled" - assert not any("[DisabledFunction][Exit]" in msg for msg in log_messages), "Exit should not be logged when disabled" - # Coherence:OK should still be logged (internal tracking) - assert any("[DisabledFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence should still be logged" + + # REASON entry marker should NOT be logged when disabled + reason_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction' + ] + assert len(reason_records) == 0, "REASON entry should not be logged when disabled" + + # REFLECT coherence marker should still be logged (internal tracking) + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction' + ] + assert len(reflect_records) >= 1, "REFLECT coherence should still be logged" # [/DEF:test_enable_belief_state_flag:Function] @@ -257,7 +298,7 @@ def test_configure_logger_post_conditions(tmp_path): import logging from logging.handlers import RotatingFileHandler from src.core.config_models import LoggingConfig - from src.core.logger import configure_logger, logger, BeliefFormatter, get_task_log_level + from src.core.logger import configure_logger, logger, CotJsonFormatter, get_task_log_level import src.core.logger as logger_module log_file = tmp_path / "test.log" @@ -267,25 +308,93 @@ def test_configure_logger_post_conditions(tmp_path): enable_belief_state=False, file_path=str(log_file) ) - + configure_logger(config) - + # 1. Logger level is updated assert logger.level == logging.WARNING - + # 2. Handlers are updated (file handler removed old ones, added new one) file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)] assert len(file_handlers) == 1 import pathlib assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve() - - # 3. Formatter is set to BeliefFormatter + + # 3. Formatter is set to CotJsonFormatter for handler in logger.handlers: - assert isinstance(handler.formatter, BeliefFormatter) - + assert isinstance(handler.formatter, CotJsonFormatter) + # 4. Global states assert getattr(logger_module, '_enable_belief_state') is False assert get_task_log_level() == "DEBUG" # [/DEF:test_configure_logger_post_conditions:Function] +# [DEF:test_cot_json_formatter_output:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that CotJsonFormatter produces valid JSON with expected fields. +def test_cot_json_formatter_output(): + """Test that CotJsonFormatter produces valid JSON with expected fields.""" + import json + from datetime import datetime, timezone + + formatter = CotJsonFormatter() + + # Create a LogRecord with structured extra data + record = logging.LogRecord( + name="test.module", + level=logging.INFO, + pathname="/fake/path.py", + lineno=42, + msg="Test action", + args=(), + exc_info=None, + ) + record.marker = "REASON" + record.intent = "Test action" + record.src = "TestModule" + record.payload = {"key": "value"} + + output = formatter.format(record) + parsed = json.loads(output) + + assert parsed["level"] == "INFO" + assert parsed["marker"] == "REASON" + assert parsed["intent"] == "Test action" + assert parsed["src"] == "TestModule" + assert parsed["payload"] == {"key": "value"} + assert "ts" in parsed + assert "trace_id" in parsed +# [/DEF:test_cot_json_formatter_output:Function] + +# [DEF:test_cot_json_formatter_plain_message:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that CotJsonFormatter wraps plain messages (no extra) with default marker. +def test_cot_json_formatter_plain_message(): + """Test that CotJsonFormatter wraps plain messages with default marker.""" + import json + + formatter = CotJsonFormatter() + + # Create a LogRecord WITHOUT extra data (plain message) + record = logging.LogRecord( + name="test.module", + level=logging.INFO, + pathname="/fake/path.py", + lineno=42, + msg="Plain info message", + args=(), + exc_info=None, + ) + + output = formatter.format(record) + parsed = json.loads(output) + + assert parsed["level"] == "INFO" + assert parsed["marker"] == "REASON" # default for plain messages + assert parsed["intent"] == "Plain info message" + assert parsed["src"] == "test.module" + assert "ts" in parsed + assert "trace_id" in parsed +# [/DEF:test_cot_json_formatter_plain_message:Function] + # [/DEF:test_logger:Module] diff --git a/backend/src/core/mapping_service.py b/backend/src/core/mapping_service.py index 816af9ba..96bf86fe 100644 --- a/backend/src/core/mapping_service.py +++ b/backend/src/core/mapping_service.py @@ -21,7 +21,10 @@ from sqlalchemy.orm import Session from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger from src.models.mapping import ResourceMapping, ResourceType +from src.core.cot_logger import MarkerLogger from src.core.logger import logger, belief_scope + +log = MarkerLogger("IdMapping") # [/SECTION] @@ -71,9 +74,7 @@ class IdMappingService: with belief_scope("IdMappingService.start_scheduler"): if self._sync_job: self.scheduler.remove_job(self._sync_job.id) - logger.info( - "[IdMappingService.start_scheduler][Reflect] Removed existing sync job." - ) + log.reflect("Removed existing sync job.") def sync_all(): for env_id in environments: @@ -90,13 +91,9 @@ class IdMappingService: if not self.scheduler.running: self.scheduler.start() - logger.info( - f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}" - ) + log.reason(f"Started background scheduler with cron: {cron_string}") else: - logger.info( - f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}" - ) + log.reason(f"Updated background scheduler with cron: {cron_string}") # [/DEF:start_scheduler:Function] @@ -114,9 +111,7 @@ class IdMappingService: If incremental=True, only fetches items changed since the max last_synced_at date. """ with belief_scope("IdMappingService.sync_environment"): - logger.info( - f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})" - ) + log.reason(f"Starting sync for environment {environment_id} (incremental={incremental})") # Implementation Note: In a real scenario, superset_client needs to be an instance # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/ @@ -136,9 +131,7 @@ class IdMappingService: total_deleted = 0 try: for res_enum, endpoint, name_field in types_to_poll: - logger.debug( - f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint" - ) + log.reason(f"Polling {endpoint} endpoint") # Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... ) # This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] } @@ -163,8 +156,8 @@ class IdMappingService: from datetime import timedelta since_dttm = max_date - timedelta(minutes=5) - logger.debug( - f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}" + log.reason( + f"Incremental sync since {since_dttm}" ) resources = superset_client.get_all_resources( @@ -228,26 +221,18 @@ class IdMappingService: deleted = stale_query.delete(synchronize_session="fetch") if deleted: total_deleted += deleted - logger.info( - f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}" - ) + log.reason(f"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}") except Exception as loop_e: - logger.error( - f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}" - ) + log.explore(f"Error polling {endpoint}", error=str(loop_e)) # Continue to next resource type instead of blowing up the whole sync self.db.commit() - logger.info( - f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items." - ) + log.reflect(f"Successfully synced {total_synced} items and deleted {total_deleted} stale items.") except Exception as e: self.db.rollback() - logger.error( - f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}" - ) + log.explore("Critical sync failure", error=str(e)) raise # [/DEF:sync_environment:Function] diff --git a/backend/src/core/middleware/__init__.py b/backend/src/core/middleware/__init__.py new file mode 100644 index 00000000..60c85c06 --- /dev/null +++ b/backend/src/core/middleware/__init__.py @@ -0,0 +1,5 @@ +# [DEF:middleware_package:Package] +# @COMPLEXITY: 1 +# @SEMANTICS: middleware, package, starlette, fastapi +# @PURPOSE: FastAPI/Starlette middleware package for request-level context and tracing. +# [/DEF:middleware_package:Package] diff --git a/backend/src/core/middleware/trace.py b/backend/src/core/middleware/trace.py new file mode 100644 index 00000000..7fff9aa9 --- /dev/null +++ b/backend/src/core/middleware/trace.py @@ -0,0 +1,77 @@ +# [DEF:TraceContextMiddlewareModule:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: middleware, trace, context, starlette, fastapi +# @PURPOSE: FastAPI/Starlette middleware that seeds a trace_id for every incoming HTTP request. +# Optionally extracts X-Trace-ID header for cross-service trace propagation. +# @LAYER: Core +# @RELATION: [DEPENDS_ON] -> [CotLoggerModule] +# @RELATION: [CALLED_BY] -> [AppModule] +# @PRE: FastAPI app instance with Starlette middleware support. +# @POST: Every request gets a trace_id via seed_trace_id(). Existing X-Trace-ID header is +# preserved and used as the trace_id when present. +# @SIDE_EFFECT: Sets ContextVar _trace_id for the duration of the request. + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response +from starlette.types import ASGIApp + +from ..cot_logger import seed_trace_id, set_trace_id + +# [DEF:TraceContextMiddleware:Class] +# @COMPLEXITY: 2 +# @SEMANTICS: middleware, trace, context, dispatch +# @BRIEF: Starlette BaseHTTPMiddleware that seeds a trace_id per request. +class TraceContextMiddleware(BaseHTTPMiddleware): + """FastAPI/Starlette middleware that seeds a trace_id for every request. + + If the incoming request carries an ``X-Trace-ID`` header, that value is + used as the trace_id (enabling cross-service trace chaining). Otherwise a + new UUID4 is generated. + + Usage:: + + from backend.src.core.middleware.trace import TraceContextMiddleware + app.add_middleware(TraceContextMiddleware) + """ + + # [DEF:TraceContextMiddleware.__init__:Function] + # @BRIEF: Standard BaseHTTPMiddleware initialiser. + def __init__(self, app: ASGIApp) -> None: + super().__init__(app) + + # [/DEF:TraceContextMiddleware.__init__:Function] + + # [DEF:TraceContextMiddleware.dispatch:Function] + # @COMPLEXITY: 2 + # @SEMANTICS: dispatch, trace, seed, header, call_next + # @BRIEF: Dispatch handler that seeds trace_id before passing to the next middleware. + async def dispatch( + self, request: Request, call_next + ) -> Response: + """Intercept every request, seed the trace_id, and forward. + + If the client sent an ``X-Trace-ID`` header, use it; otherwise + ``seed_trace_id()`` generates a new UUID4. + + Args: + request: The incoming Starlette Request. + call_next: The next middleware or route handler. + + Returns: + Response from the downstream handler. + """ + incoming_trace_id = request.headers.get("X-Trace-ID") + if incoming_trace_id: + set_trace_id(incoming_trace_id) + else: + seed_trace_id() + + response = await call_next(request) + return response + + # [/DEF:TraceContextMiddleware.dispatch:Function] + + +# [/DEF:TraceContextMiddleware:Class] +# [/DEF:TraceContextMiddlewareModule:Module] diff --git a/backend/src/core/migration/risk_assessor.py b/backend/src/core/migration/risk_assessor.py index 916cdb06..3823b9ef 100644 --- a/backend/src/core/migration/risk_assessor.py +++ b/backend/src/core/migration/risk_assessor.py @@ -28,9 +28,12 @@ from typing import Any, Dict, List -from ..logger import logger, belief_scope +from ..logger import belief_scope +from ..cot_logger import MarkerLogger from ..superset_client import SupersetClient +log = MarkerLogger("RiskAssessor") + # [DEF:index_by_uuid:Function] # @PURPOSE: Build UUID-index from normalized objects. @@ -40,13 +43,13 @@ from ..superset_client import SupersetClient # @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]] def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: with belief_scope("risk_assessor.index_by_uuid"): - logger.reason("Building UUID index", extra={"objects_count": len(objects)}) + log.reason("Building UUID index", payload={"objects_count": len(objects)}) indexed: Dict[str, Dict[str, Any]] = {} for obj in objects: uuid = obj.get("uuid") if uuid: indexed[str(uuid)] = obj - logger.reflect("UUID index built", extra={"indexed_count": len(indexed)}) + log.reflect("UUID index built", payload={"indexed_count": len(indexed)}) return indexed @@ -61,9 +64,9 @@ def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: # @DATA_CONTRACT: Any -> List[str] def extract_owner_identifiers(owners: Any) -> List[str]: with belief_scope("risk_assessor.extract_owner_identifiers"): - logger.reason("Normalizing owner identifiers") + log.reason("Normalizing owner identifiers") if not isinstance(owners, list): - logger.reflect("Owners payload is not list; returning empty identifiers") + log.reflect("Owners payload is not list; returning empty identifiers") return [] ids: List[str] = [] for owner in owners: @@ -75,8 +78,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]: elif owner is not None: ids.append(str(owner)) normalized_ids = sorted(set(ids)) - logger.reflect( - "Owner identifiers normalized", extra={"owner_count": len(normalized_ids)} + log.reflect( + "Owner identifiers normalized", payload={"owner_count": len(normalized_ids)} ) return normalized_ids @@ -105,7 +108,7 @@ def build_risks( target_client: SupersetClient, ) -> List[Dict[str, Any]]: with belief_scope("risk_assessor.build_risks"): - logger.reason("Building migration risks from diff payload") + log.reason("Building migration risks from diff payload") risks: List[Dict[str, Any]] = [] for object_type in ("dashboards", "charts", "datasets"): for item in diff[object_type]["update"]: @@ -170,7 +173,7 @@ def build_risks( "message": f"Owner mismatch for dashboard {item.get('title') or item['uuid']}", } ) - logger.reflect("Risk list assembled", extra={"risk_count": len(risks)}) + log.reflect("Risk list assembled", payload={"risk_count": len(risks)}) return risks @@ -185,14 +188,14 @@ def build_risks( # @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any] def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]: with belief_scope("risk_assessor.score_risks"): - logger.reason("Scoring risk items", extra={"risk_items_count": len(risk_items)}) + log.reason("Scoring risk items", payload={"risk_items_count": len(risk_items)}) weights = {"high": 25, "medium": 10, "low": 5} score = min( 100, sum(weights.get(item.get("severity", "low"), 5) for item in risk_items) ) level = "low" if score < 25 else "medium" if score < 60 else "high" result = {"score": score, "level": level, "items": risk_items} - logger.reflect("Risk score computed", extra={"score": score, "level": level}) + log.reflect("Risk score computed", payload={"score": score, "level": level}) return result diff --git a/backend/src/core/scheduler.py b/backend/src/core/scheduler.py index 4df85b87..d089b7c5 100644 --- a/backend/src/core/scheduler.py +++ b/backend/src/core/scheduler.py @@ -10,6 +10,10 @@ from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger from .logger import logger, belief_scope from .config_manager import ConfigManager +from .cot_logger import MarkerLogger, get_trace_id, set_trace_id + +log = MarkerLogger("SchedulerService") +log_tsc = MarkerLogger("ThrottledSchedulerConfigurator") import asyncio from datetime import datetime, time, timedelta, date # [/SECTION] @@ -42,7 +46,7 @@ class SchedulerService: with belief_scope("SchedulerService.start"): if not self.scheduler.running: self.scheduler.start() - logger.info("Scheduler started.") + log.reflect("Scheduler started") self.load_schedules() # [/DEF:start:Function] @@ -55,7 +59,7 @@ class SchedulerService: with belief_scope("SchedulerService.stop"): if self.scheduler.running: self.scheduler.shutdown() - logger.info("Scheduler stopped.") + log.reflect("Scheduler stopped") # [/DEF:stop:Function] @@ -95,10 +99,12 @@ class SchedulerService: args=[env_id], replace_existing=True, ) - logger.info( - f"Scheduled backup job added for environment {env_id}: {cron_expression}" + log.reflect( + "Scheduled backup job added", + payload={"env_id": env_id, "cron": cron_expression}, ) except Exception as e: + log.explore("Failed to add backup job", error=str(e), payload={"env_id": env_id, "cron": cron_expression}) logger.error(f"Failed to add backup job for environment {env_id}: {e}") # [/DEF:add_backup_job:Function] @@ -110,7 +116,7 @@ class SchedulerService: # @PARAM: env_id (str) - The ID of the environment. def _trigger_backup(self, env_id: str): with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"): - logger.info(f"Triggering scheduled backup for environment {env_id}") + log.reason("Triggering scheduled backup", payload={"env_id": env_id}) # Check if a backup is already running for this environment active_tasks = self.task_manager.get_tasks(limit=100) @@ -120,19 +126,21 @@ class SchedulerService: and task.status in ["PENDING", "RUNNING"] and task.params.get("environment_id") == env_id ): - logger.warning( - f"Backup already running for environment {env_id}. Skipping scheduled run." - ) + log.explore("Backup already running, skipping scheduled run", error="Duplicate backup prevented", payload={"env_id": env_id}) return # Run the backup task # We need to run this in the event loop since create_task is async - asyncio.run_coroutine_threadsafe( - self.task_manager.create_task( + trace_id = get_trace_id() + + async def _backup_task(): + if trace_id: + set_trace_id(trace_id) + await self.task_manager.create_task( "superset-backup", {"environment_id": env_id} - ), - self.loop, - ) + ) + + asyncio.run_coroutine_threadsafe(_backup_task(), self.loop) # [/DEF:_trigger_backup:Function] @@ -176,9 +184,7 @@ class ThrottledSchedulerConfigurator: # Minimum interval of 1 second to avoid division by zero or negative if total_seconds <= 0: - logger.warning( - f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks." - ) + log_tsc.explore("Window size is zero or negative, falling back to start time", error=f"total_seconds={total_seconds}", payload={"n": n}) return [start_dt] * n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds), @@ -193,9 +199,7 @@ class ThrottledSchedulerConfigurator: # If interval is too small (e.g. < 1s), we might want a fallback, # but the spec says "handle too-small windows with explicit fallback/warning". if interval < 1: - logger.warning( - f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated." - ) + log_tsc.explore("Window too small for task distribution", error=f"interval={interval:.2f}s", payload={"n": n}) scheduled_times = [] for i in range(n): diff --git a/backend/src/core/superset_client/_base.py b/backend/src/core/superset_client/_base.py index de2d2264..9cffc1bf 100644 --- a/backend/src/core/superset_client/_base.py +++ b/backend/src/core/superset_client/_base.py @@ -14,12 +14,13 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union, cast from requests import Response -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope from ..utils.network import APIClient, SupersetAPIError from ..utils.fileio import get_filename_from_headers from ..config_models import Environment -app_logger = cast(Any, app_logger) +log = MarkerLogger("SupersetClientBase") # [DEF:SupersetClientBase:Class] @@ -36,9 +37,9 @@ class SupersetClientBase: # @RELATION: DEPENDS_ON -> [APIClient] def __init__(self, env: Environment): with belief_scope("SupersetClientInit"): - app_logger.reason( + log.reason( "Initializing Superset client for environment", - extra={"environment": getattr(env, "id", None), "env_name": env.name}, + payload={"environment": getattr(env, "id", None), "env_name": env.name}, ) self.env = env # Construct auth payload expected by Superset API @@ -54,9 +55,9 @@ class SupersetClientBase: timeout=env.timeout, ) self.delete_before_reimport: bool = False - app_logger.reflect( + log.reflect( "Superset client initialized", - extra={"environment": getattr(self.env, "id", None)}, + payload={"environment": getattr(self.env, "id", None)}, ) # [/DEF:SupersetClientInit:Function] @@ -67,14 +68,14 @@ class SupersetClientBase: # @RELATION: CALLS -> [APIClient] def authenticate(self) -> Dict[str, str]: with belief_scope("SupersetClientAuthenticate"): - app_logger.reason( + log.reason( "Authenticating Superset client", - extra={"environment": getattr(self.env, "id", None)}, + payload={"environment": getattr(self.env, "id", None)}, ) tokens = self.network.authenticate() - app_logger.reflect( + log.reflect( "Superset client authentication completed", - extra={ + payload={ "environment": getattr(self.env, "id", None), "token_keys": sorted(tokens.keys()), }, @@ -140,12 +141,10 @@ class SupersetClientBase: # @RELATION: CALLS -> [APIClient] def _do_import(self, file_name: Union[str, Path]) -> Dict: with belief_scope("_do_import"): - app_logger.debug(f"[_do_import][State] Uploading file: {file_name}") + log.reason("Uploading file for import", payload={"file_name": str(file_name)}) file_path = Path(file_name) if not file_path.exists(): - app_logger.error( - f"[_do_import][Failure] File does not exist: {file_name}" - ) + log.explore("Import file does not exist", error="FileNotFound", payload={"file_name": str(file_name)}) raise FileNotFoundError(f"File does not exist: {file_name}") return self.network.upload_file( @@ -187,9 +186,9 @@ class SupersetClientBase: timestamp = datetime.now().strftime("%Y%m%dT%H%M%S") filename = f"dashboard_export_{dashboard_id}_{timestamp}.zip" - app_logger.warning( - "[_resolve_export_filename][Warning] Generated filename: %s", - filename, + log.reflect( + "Export filename not found in headers, using generated name", + payload={"dashboard_id": dashboard_id, "generated_filename": filename}, ) return filename @@ -224,10 +223,7 @@ class SupersetClientBase: if dash_id is not None: return dash_id if dash_slug is not None: - app_logger.debug( - "[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.", - dash_slug, - ) + log.reason("Resolving dashboard ID by slug", payload={"slug": dash_slug}) try: _, candidates = self.get_dashboards( query={ @@ -236,17 +232,10 @@ class SupersetClientBase: ) if candidates: target_id = candidates[0]["id"] - app_logger.debug( - "[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.", - target_id, - ) + log.reflect("Resolved slug to dashboard ID", payload={"slug": dash_slug, "target_id": target_id}) return target_id except Exception as e: - app_logger.warning( - "[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s", - dash_slug, - e, - ) + log.explore("Could not resolve slug to dashboard ID", error=str(e), payload={"slug": dash_slug}) return None # [/DEF:SupersetClientResolveTargetIdForDelete:Function] @@ -278,10 +267,7 @@ class SupersetClientBase: } config = column_map.get(resource_type) if not config: - app_logger.warning( - "[get_all_resources][Warning] Unknown resource type: %s", - resource_type, - ) + log.explore("Unknown resource type requested", error=f"Invalid resource type: {resource_type}", payload={"resource_type": resource_type}) return [] query = {"columns": config["columns"]} @@ -301,10 +287,9 @@ class SupersetClientBase: endpoint=config["endpoint"], pagination_options={"base_query": validated, "results_field": "result"}, ) - app_logger.info( - "[get_all_resources][Exit] Fetched %d %s resources.", - len(data), - resource_type, + log.reflect( + "Resources fetched", + payload={"resource_type": resource_type, "count": len(data)}, ) return data diff --git a/backend/src/core/superset_client/_dashboards_crud.py b/backend/src/core/superset_client/_dashboards_crud.py index 84f25481..81580d93 100644 --- a/backend/src/core/superset_client/_dashboards_crud.py +++ b/backend/src/core/superset_client/_dashboards_crud.py @@ -13,9 +13,10 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union, cast from requests import Response -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope -app_logger = cast(Any, app_logger) +log = MarkerLogger("SupersetDashboardsCrud") # [DEF:SupersetDashboardsCrudMixin:Class] @@ -107,9 +108,7 @@ class SupersetDashboardsCrudMixin: or "Chart", }) except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s", e, - ) + log.explore("Failed to fetch dashboard charts", error=str(e), payload={"dashboard_ref": dashboard_ref}) try: datasets_response = self.network.request( @@ -147,9 +146,7 @@ class SupersetDashboardsCrudMixin: "overview": fq_name, }) except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s", e, - ) + log.explore("Failed to fetch dashboard datasets", error=str(e), payload={"dashboard_ref": dashboard_ref}) # Fallback: derive chart IDs from layout metadata if not charts: @@ -182,9 +179,12 @@ class SupersetDashboardsCrudMixin: self._extract_chart_ids_from_layout(raw_json_metadata) ) - app_logger.info( - "[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)", - len(chart_ids_from_position), dashboard_ref, + log.reason( + "Extracted fallback chart IDs from layout", + payload={ + "chart_count": len(chart_ids_from_position), + "dashboard_ref": dashboard_ref, + }, ) for chart_id in sorted(chart_ids_from_position): @@ -202,10 +202,7 @@ class SupersetDashboardsCrudMixin: or chart_data.get("viz_type") or "Chart", }) except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s", - chart_id, e, - ) + log.explore("Failed to resolve fallback chart", error=str(e), payload={"chart_id": chart_id}) # Backfill datasets from chart datasource IDs. dataset_ids_from_charts = { @@ -247,10 +244,7 @@ class SupersetDashboardsCrudMixin: "overview": fq_name, }) except Exception as e: - app_logger.warning( - "[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s", - dataset_id, e, - ) + log.explore("Failed to resolve dataset from chart datasource", error=str(e), payload={"dataset_id": dataset_id}) unique_charts = {chart["id"]: chart for chart in charts} unique_datasets = {dataset["id"]: dataset for dataset in datasets} @@ -281,9 +275,7 @@ class SupersetDashboardsCrudMixin: # @RELATION: CALLS -> [APIClient] def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]: with belief_scope("export_dashboard"): - app_logger.info( - "[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id - ) + log.reason("Exporting dashboard", payload={"dashboard_id": dashboard_id}) response = self.network.request( method="GET", endpoint="/dashboard/export/", @@ -294,10 +286,7 @@ class SupersetDashboardsCrudMixin: response = cast(Response, response) self._validate_export_response(response, dashboard_id) filename = self._resolve_export_filename(response, dashboard_id) - app_logger.info( - "[export_dashboard][Exit] Exported dashboard %s to %s.", - dashboard_id, filename, - ) + log.reflect("Dashboard exported", payload={"dashboard_id": dashboard_id, "filename": filename}) return response.content, filename # [/DEF:SupersetClientExportDashboard:Function] @@ -322,24 +311,19 @@ class SupersetDashboardsCrudMixin: try: return self._do_import(file_path) except Exception as exc: - app_logger.error( - "[import_dashboard][Failure] First import attempt failed: %s", - exc, exc_info=True, - ) + log.explore("First import attempt failed", error=str(exc), payload={"file_name": file_name}) if not self.delete_before_reimport: raise target_id = self._resolve_target_id_for_delete(dash_id, dash_slug) if target_id is None: - app_logger.error( - "[import_dashboard][Failure] No ID available for delete-retry." - ) + log.explore("No ID available for delete-retry during import", error="Cannot delete-retry: neither dash_id nor dash_slug resolved", payload={"dash_id": dash_id, "dash_slug": dash_slug}) raise self.delete_dashboard(target_id) - app_logger.info( - "[import_dashboard][State] Deleted dashboard ID %s, retrying import.", - target_id, + log.reason( + "Deleted dashboard, retrying import", + payload={"target_id": target_id}, ) return self._do_import(file_path) @@ -352,22 +336,15 @@ class SupersetDashboardsCrudMixin: # @RELATION: CALLS -> [APIClient] def delete_dashboard(self, dashboard_id: Union[int, str]) -> None: with belief_scope("delete_dashboard"): - app_logger.info( - "[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id - ) + log.reason("Deleting dashboard", payload={"dashboard_id": dashboard_id}) response = self.network.request( method="DELETE", endpoint=f"/dashboard/{dashboard_id}" ) response = cast(Dict, response) if response.get("result", True) is not False: - app_logger.info( - "[delete_dashboard][Success] Dashboard %s deleted.", dashboard_id - ) + log.reflect("Dashboard deleted", payload={"dashboard_id": dashboard_id}) else: - app_logger.warning( - "[delete_dashboard][Warning] Unexpected response while deleting %s: %s", - dashboard_id, response, - ) + log.explore("Unexpected response while deleting dashboard", error=f"Unexpected API response: {response}", payload={"dashboard_id": dashboard_id, "response": response}) # [/DEF:SupersetClientDeleteDashboard:Function] diff --git a/backend/src/core/superset_client/_dashboards_filters.py b/backend/src/core/superset_client/_dashboards_filters.py index 070d51f9..5d733b21 100644 --- a/backend/src/core/superset_client/_dashboards_filters.py +++ b/backend/src/core/superset_client/_dashboards_filters.py @@ -8,9 +8,10 @@ import json from typing import Any, Dict, Optional, Union, cast -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope -app_logger = cast(Any, app_logger) +log = MarkerLogger("SupersetDashboardsFilters") # [DEF:SupersetDashboardsFiltersMixin:Class] @@ -122,10 +123,7 @@ class SupersetDashboardsFiltersMixin: try: parsed_value = json.loads(value) except json.JSONDecodeError as e: - app_logger.warning( - "[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s", - e, - ) + log.explore("Failed to parse filter state JSON", error=str(e)) parsed_value = {} elif isinstance(value, dict): parsed_value = value @@ -226,11 +224,7 @@ class SupersetDashboardsFiltersMixin: if raw_id is not None: resolved_id = int(raw_id) except Exception as e: - app_logger.warning( - "[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s", - dashboard_ref, - e, - ) + log.explore("Failed to resolve dashboard slug to ID", error=str(e), payload={"slug": dashboard_ref}) if resolved_id is not None: filter_data = self.extract_native_filters_from_key( @@ -241,9 +235,7 @@ class SupersetDashboardsFiltersMixin: result["filters"] = filter_data return result else: - app_logger.warning( - "[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key" - ) + log.explore("Could not resolve dashboard_id from URL for native_filters_key", error="Dashboard ID could not be resolved from URL", payload={"dashboard_ref": dashboard_ref}) # Check for native_filters in query params (direct filter values) native_filters = query_params.get("native_filters", [None])[0] @@ -254,10 +246,7 @@ class SupersetDashboardsFiltersMixin: result["filters"] = {"dataMask": parsed_filters} return result except json.JSONDecodeError as e: - app_logger.warning( - "[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s", - e, - ) + log.explore("Failed to parse native_filters JSON", error=str(e)) return result diff --git a/backend/src/core/superset_client/_dashboards_list.py b/backend/src/core/superset_client/_dashboards_list.py index 3f0c70c3..50c4c407 100644 --- a/backend/src/core/superset_client/_dashboards_list.py +++ b/backend/src/core/superset_client/_dashboards_list.py @@ -9,9 +9,10 @@ import json from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope -app_logger = cast(Any, app_logger) +log = MarkerLogger("SupersetDashboardsList") # [DEF:SupersetDashboardsListMixin:Class] @@ -26,7 +27,7 @@ class SupersetDashboardsListMixin: # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_dashboards"): - app_logger.info("[get_dashboards][Enter] Fetching dashboards.") + log.reason("Fetching dashboards", payload={"has_query": query is not None}) validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: validated_query["columns"] = [ @@ -42,7 +43,7 @@ class SupersetDashboardsListMixin: }, ) total_count = len(paginated_data) - app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count) + log.reflect("Dashboards fetched", payload={"total_count": total_count}) return total_count, paginated_data # [/DEF:SupersetClientGetDashboards:Function] @@ -127,19 +128,29 @@ class SupersetDashboardsListMixin: }) if index < max_debug_samples: - app_logger.reflect( - "[REFLECT] Dashboard actor projection sample " - f"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, " - f"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, " - f"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, " - f"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, " - f"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})" + log.reflect( + "Dashboard actor projection sample", + payload={ + "env": getattr(self.env, "id", None), + "dashboard_id": dash.get("id"), + "raw_owners": raw_owners, + "raw_owner_usernames": raw_owner_usernames, + "raw_created_by": raw_created_by, + "raw_changed_by": raw_changed_by, + "raw_changed_by_name": raw_changed_by_name, + "projected_owners": owners, + "projected_created_by": projected_created_by, + "projected_modified_by": projected_modified_by, + }, ) - app_logger.reflect( - "[REFLECT] Dashboard actor projection summary " - f"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, " - f"sampled={min(len(result), max_debug_samples)})" + log.reflect( + "Dashboard actor projection summary", + payload={ + "env": getattr(self.env, "id", None), + "dashboards": len(result), + "sampled": min(len(result), max_debug_samples), + }, ) return result diff --git a/backend/src/core/superset_client/_databases.py b/backend/src/core/superset_client/_databases.py index b1fe6141..3687d154 100644 --- a/backend/src/core/superset_client/_databases.py +++ b/backend/src/core/superset_client/_databases.py @@ -7,9 +7,10 @@ from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope -app_logger = cast(Any, app_logger) +log = MarkerLogger("SupersetDatabases") # [DEF:SupersetDatabasesMixin:Class] @@ -23,7 +24,7 @@ class SupersetDatabasesMixin: # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_databases"): - app_logger.info("[get_databases][Enter] Fetching databases.") + log.reason("Fetching databases", payload={"has_query": query is not None}) validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: validated_query["columns"] = [] @@ -36,7 +37,7 @@ class SupersetDatabasesMixin: }, ) total_count = len(paginated_data) - app_logger.info("[get_databases][Exit] Found %d databases.", total_count) + log.reflect("Databases fetched", payload={"total_count": total_count}) return total_count, paginated_data # [/DEF:SupersetClientGetDatabases:Function] @@ -47,12 +48,12 @@ class SupersetDatabasesMixin: # @RELATION: CALLS -> [APIClient] def get_database(self, database_id: int) -> Dict: with belief_scope("get_database"): - app_logger.info("[get_database][Enter] Fetching database %s.", database_id) + log.reason("Fetching database", payload={"database_id": database_id}) response = self.network.request( method="GET", endpoint=f"/database/{database_id}" ) response = cast(Dict, response) - app_logger.info("[get_database][Exit] Got database %s.", database_id) + log.reflect("Database fetched", payload={"database_id": database_id}) return response # [/DEF:SupersetClientGetDatabase:Function] diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index 11ac0bd3..863d2c37 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -8,9 +8,10 @@ import json from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope -app_logger = cast(Any, app_logger) +log = MarkerLogger("SupersetDatasets") # [DEF:SupersetDatasetsMixin:Class] @@ -24,7 +25,7 @@ class SupersetDatasetsMixin: # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_datasets"): - app_logger.info("[get_datasets][Enter] Fetching datasets.") + log.reason("Fetching datasets") validated_query = self._validate_query_params(query) paginated_data = self._fetch_all_pages( @@ -35,7 +36,7 @@ class SupersetDatasetsMixin: }, ) total_count = len(paginated_data) - app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count) + log.reflect("Datasets fetched", payload={"total_count": total_count}) return total_count, paginated_data # [/DEF:SupersetClientGetDatasets:Function] @@ -144,9 +145,7 @@ class SupersetDatasetsMixin: {"id": dash_id, "title": f"Dashboard {dash_id}", "slug": None} ) except Exception as e: - app_logger.warning( - f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}" - ) + log.explore("Failed to fetch related dashboards", error=str(e), payload={"dataset_id": dataset_id}) linked_dashboards = [] database_obj = dataset.get("database") @@ -171,8 +170,13 @@ class SupersetDatasetsMixin: "changed_on": dataset.get("changed_on"), } - app_logger.info( - f"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards" + log.reflect( + "Dataset detail fetched", + payload={ + "dataset_id": dataset_id, + "column_count": len(column_info), + "linked_dashboard_count": len(linked_dashboards), + }, ) return result @@ -184,12 +188,12 @@ class SupersetDatasetsMixin: # @RELATION: CALLS -> [APIClient] def get_dataset(self, dataset_id: int) -> Dict: with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"): - app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id) + log.reason("Fetching dataset", payload={"dataset_id": dataset_id}) response = self.network.request( method="GET", endpoint=f"/dataset/{dataset_id}" ) response = cast(Dict, response) - app_logger.info("[get_dataset][Exit] Got dataset %s.", dataset_id) + log.reflect("Dataset fetched", payload={"dataset_id": dataset_id}) return response # [/DEF:SupersetClientGetDataset:Function] @@ -201,7 +205,7 @@ class SupersetDatasetsMixin: # @RELATION: CALLS -> [APIClient] def update_dataset(self, dataset_id: int, data: Dict) -> Dict: with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"): - app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id) + log.reason("Updating dataset", payload={"dataset_id": dataset_id}) response = self.network.request( method="PUT", endpoint=f"/dataset/{dataset_id}", @@ -209,7 +213,7 @@ class SupersetDatasetsMixin: headers={"Content-Type": "application/json"}, ) response = cast(Dict, response) - app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id) + log.reflect("Dataset updated", payload={"dataset_id": dataset_id}) return response # [/DEF:SupersetClientUpdateDataset:Function] diff --git a/backend/src/core/superset_client/_datasets_preview.py b/backend/src/core/superset_client/_datasets_preview.py index b0cff164..ee193172 100644 --- a/backend/src/core/superset_client/_datasets_preview.py +++ b/backend/src/core/superset_client/_datasets_preview.py @@ -10,8 +10,12 @@ import json from copy import deepcopy from typing import Any, Dict, List, Optional -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope from ..utils.network import SupersetAPIError + +log = MarkerLogger("SupersetDatasetsPreview") + # [DEF:SupersetDatasetsPreviewMixin:Class] # @COMPLEXITY: 4 # @PURPOSE: Mixin providing dataset preview compilation and query context building. @@ -21,7 +25,7 @@ class SupersetDatasetsPreviewMixin: # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output. def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: with belief_scope('SupersetClientCompileDatasetPreview'): - app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview') + log.reason('Compiling dataset preview SQL', payload={"dataset_id": dataset_id}) dataset_response = self.get_dataset(dataset_id) dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {} query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or []) @@ -49,7 +53,7 @@ class SupersetDatasetsPreviewMixin: elif isinstance(request_body, dict): request_payload_keys = sorted(request_body.keys()) strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys} - app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])}) + log.reason('Attempting dataset preview compilation strategy', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])}) try: response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None) normalized = self._extract_compiled_sql_from_preview_response(response) @@ -59,13 +63,12 @@ class SupersetDatasetsPreviewMixin: normalized['endpoint_kind'] = endpoint_kind normalized['dataset_id'] = dataset_id normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}] - app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')}) - app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview') + log.reflect('Dataset preview compilation returned normalized SQL payload', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')}) return normalized except Exception as exc: failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)} strategy_attempts.append(failure_diagnostics) - app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}) + log.explore('Dataset preview compilation strategy failed', payload={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}, error=str(exc)) raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})') # [/DEF:SupersetClientCompileDatasetPreview:Function] @@ -74,7 +77,7 @@ class SupersetDatasetsPreviewMixin: # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic. def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]: with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'): - app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') + log.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters) query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {}) legacy_form_data = deepcopy(query_context.get('form_data', {})) @@ -96,8 +99,8 @@ class SupersetDatasetsPreviewMixin: time_range = query_object.get('time_range') if time_range: legacy_form_data['time_range'] = time_range - app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})}) - app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') + log.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', payload={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})}) + log.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') return legacy_form_data # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function] @@ -112,9 +115,9 @@ class SupersetDatasetsPreviewMixin: effective_filters: List[Dict[str, Any]], ) -> Dict[str, Any]: with belief_scope("SupersetClientBuildDatasetPreviewQueryContext"): - app_logger.reason( + log.reason( "Building Superset dataset preview query context", - extra={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])}, + payload={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])}, ) normalized_template_params = deepcopy(template_params or {}) normalized_filter_payload = ( @@ -151,10 +154,7 @@ class SupersetDatasetsPreviewMixin: for key, value in parsed_dataset_template_params.items(): normalized_template_params.setdefault(str(key), value) except json.JSONDecodeError: - app_logger.explore( - "Dataset template_params could not be parsed while building preview query context", - extra={"dataset_id": dataset_id}, - ) + log.explore("Dataset template_params could not be parsed while building preview query context", error="Failed to parse template_params JSON", payload={"dataset_id": dataset_id}) extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data) if normalized_filters: @@ -210,9 +210,9 @@ class SupersetDatasetsPreviewMixin: "result_type": result_type, "force": True, } - app_logger.reflect( + log.reflect( "Built Superset dataset preview query context", - extra={ + payload={ "dataset_id": dataset_id, "datasource": datasource_payload, "normalized_effective_filters": normalized_filters, diff --git a/backend/src/core/superset_client/_datasets_preview_filters.py b/backend/src/core/superset_client/_datasets_preview_filters.py index dbe22b0a..6c24da0c 100644 --- a/backend/src/core/superset_client/_datasets_preview_filters.py +++ b/backend/src/core/superset_client/_datasets_preview_filters.py @@ -9,9 +9,11 @@ from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple -from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger +from ..logger import belief_scope from ..utils.network import SupersetAPIError +log = MarkerLogger("SupersetDatasetsPreviewFilters") # [DEF:SupersetDatasetsPreviewFiltersMixin:Class] # @COMPLEXITY: 3 @@ -100,9 +102,9 @@ class SupersetDatasetsPreviewFiltersMixin: "outgoing_clauses": deepcopy(outgoing_clauses), } ) - app_logger.reason( + log.reason( "Normalized effective preview filter for Superset query context", - extra={ + payload={ "filter_name": display_name, "used_preserved_clauses": used_preserved_clauses, "outgoing_clauses": outgoing_clauses, diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index 3644da4a..ea89dee9 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -39,6 +39,9 @@ from .models import Task, TaskStatus, LogEntry, LogFilter, LogStats from .persistence import TaskPersistenceService, TaskLogPersistenceService from .context import TaskContext from ..logger import logger, belief_scope, should_log_task_level +from ..cot_logger import MarkerLogger, get_trace_id, set_trace_id + +log = MarkerLogger("TaskManager") # [/SECTION] @@ -119,7 +122,7 @@ class TaskManager: # @PARAM: plugin_loader - The plugin loader instance. def __init__(self, plugin_loader): with belief_scope("TaskManager.__init__"): - logger.reason("Initializing task manager runtime services") + log.reason("Initializing task manager runtime services") self.plugin_loader = plugin_loader self.tasks: Dict[str, Task] = {} self.subscribers: Dict[str, List[asyncio.Queue]] = {} @@ -148,9 +151,9 @@ class TaskManager: # Load persisted tasks on startup self.load_persisted_tasks() - logger.reflect( + log.reflect( "Task manager runtime initialized", - extra={"task_count": len(self.tasks)}, + payload={"task_count": len(self.tasks)}, ) # [/DEF:__init__:Function] @@ -189,8 +192,9 @@ class TaskManager: if logs: try: self.log_persistence_service.add_logs(task_id, logs) - logger.debug(f"Flushed {len(logs)} logs for task {task_id}") + log.reflect("Flushed logs to database", payload={"task_id": task_id, "count": len(logs)}) except Exception as e: + log.explore("Failed to flush logs to database", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to flush logs for task {task_id}: {e}") # Re-add logs to buffer on failure with self._log_buffer_lock: @@ -217,7 +221,9 @@ class TaskManager: if logs: try: self.log_persistence_service.add_logs(task_id, logs) + log.reflect("Flushed task logs immediately", payload={"task_id": task_id, "count": len(logs)}) except Exception as e: + log.explore("Failed to flush task logs immediately", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to flush logs for task {task_id}: {e}") # [/DEF:_flush_task_logs:Function] @@ -240,26 +246,26 @@ class TaskManager: ) -> Task: with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"): if not self.plugin_loader.has_plugin(plugin_id): - logger.error(f"Plugin with ID '{plugin_id}' not found.") + log.explore("Plugin not found", error=f"Plugin with ID '{plugin_id}' not found.") raise ValueError(f"Plugin with ID '{plugin_id}' not found.") self.plugin_loader.get_plugin(plugin_id) if not isinstance(params, dict): - logger.error("Task parameters must be a dictionary.") + log.explore("Invalid task params type", error="Task parameters must be a dictionary.") raise ValueError("Task parameters must be a dictionary.") - logger.reason("Creating task node and scheduling execution") + log.reason("Creating task node and scheduling execution", payload={"plugin_id": plugin_id}) task = Task(plugin_id=plugin_id, params=params, user_id=user_id) self.tasks[task.id] = task self.persistence_service.persist_task(task) - logger.info(f"Task {task.id} created and scheduled for execution") + trace_id = get_trace_id() self.loop.create_task( - self._run_task(task.id) + self._run_task(task.id, trace_id=trace_id) ) # Schedule task for execution - logger.reflect( + log.reflect( "Task creation persisted and execution scheduled", - extra={"task_id": task.id, "plugin_id": plugin_id}, + payload={"task_id": task.id, "plugin_id": plugin_id}, ) return task @@ -275,17 +281,16 @@ class TaskManager: # @RELATION: [DEPENDS_ON] ->[JobLifecycle] # @RELATION: [DEPENDS_ON] ->[TaskContext] # @RELATION: [DEPENDS_ON] ->[EventBus] - async def _run_task(self, task_id: str): + async def _run_task(self, task_id: str, trace_id: str = None): + if trace_id: + set_trace_id(trace_id) with belief_scope("TaskManager._run_task", f"task_id={task_id}"): task = self.tasks[task_id] plugin = self.plugin_loader.get_plugin(task.plugin_id) - logger.reason( + log.reason( "Transitioning task to running state", - extra={"task_id": task_id, "plugin_id": task.plugin_id}, - ) - logger.info( - f"Starting execution of task {task_id} for plugin '{plugin.name}'" + payload={"task_id": task_id, "plugin_id": task.plugin_id}, ) task.status = TaskStatus.RUNNING task.started_at = datetime.utcnow() @@ -331,7 +336,6 @@ class TaskManager: self.executor, plugin.execute, params ) - logger.info(f"Task {task_id} completed successfully") task.status = TaskStatus.SUCCESS self._add_log( task_id, @@ -339,8 +343,8 @@ class TaskManager: f"Task completed successfully for plugin '{plugin.name}'", source="system", ) + log.reflect("Task completed successfully", payload={"task_id": task_id, "plugin_id": task.plugin_id}) except Exception as e: - logger.error(f"Task {task_id} failed: {e}") task.status = TaskStatus.FAILED self._add_log( task_id, @@ -349,17 +353,15 @@ class TaskManager: source="system", metadata={"error_type": type(e).__name__}, ) + log.explore("Task failed during execution", error=str(e), payload={"task_id": task_id, "plugin_id": task.plugin_id}) finally: task.finished_at = datetime.utcnow() # Flush any remaining buffered logs before persisting task self._flush_task_logs(task_id) self.persistence_service.persist_task(task) - logger.info( - f"Task {task_id} execution finished with status: {task.status}" - ) - logger.reflect( - "Task lifecycle reached persisted terminal state", - extra={"task_id": task_id, "status": str(task.status)}, + log.reflect( + "Task execution finished", + payload={"task_id": task_id, "status": str(task.status)}, ) # [/DEF:_run_task:Function] @@ -818,7 +820,7 @@ class TaskManager: if tasks_to_remove: self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove) - logger.info(f"Cleared {len(tasks_to_remove)} tasks.") + log.reflect("Tasks cleared from registry and database", payload={"count": len(tasks_to_remove)}) return len(tasks_to_remove) # [/DEF:clear_tasks:Function] diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index bb32c271..7bdaee11 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -24,6 +24,10 @@ from ...models.mapping import Environment from ..database import TasksSessionLocal from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats from ..logger import logger, belief_scope +from ..cot_logger import MarkerLogger + +log = MarkerLogger("TaskPersistence") +log_pl = MarkerLogger("TaskLogPersistence") # [/SECTION] @@ -172,6 +176,7 @@ class TaskPersistenceService: # @RELATION: [CALLS] ->[_resolve_environment_id] def persist_task(self, task: Task) -> None: with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"): + log.reason("Persisting task to database", payload={"task_id": task.id}) session: Session = TasksSessionLocal() try: record = ( @@ -208,8 +213,8 @@ class TaskPersistenceService: # Store logs as JSON, converting datetime to string record.logs = [] - for log in task.logs: - log_dict = log.dict() + for log_entry in task.logs: + log_dict = log_entry.dict() if isinstance(log_dict.get("timestamp"), datetime): log_dict["timestamp"] = log_dict["timestamp"].isoformat() # Also clean up any datetimes in context @@ -219,14 +224,16 @@ class TaskPersistenceService: # Extract error if failed if task.status == TaskStatus.FAILED: - for log in reversed(task.logs): - if log.level == "ERROR": - record.error = log.message + for log_entry in reversed(task.logs): + if log_entry.level == "ERROR": + record.error = log_entry.message break session.commit() + log.reflect("Task persisted successfully", payload={"task_id": task.id}) except Exception as e: session.rollback() + log.explore("Failed to persist task", error=str(e), payload={"task_id": task.id}) logger.error(f"Failed to persist task {task.id}: {e}") finally: session.close() @@ -242,8 +249,10 @@ class TaskPersistenceService: # @RELATION: [CALLS] ->[persist_task] def persist_tasks(self, tasks: List[Task]) -> None: with belief_scope("TaskPersistenceService.persist_tasks"): + log.reason("Persisting multiple tasks", payload={"count": len(tasks)}) for task in tasks: self.persist_task(task) + log.reflect("All tasks persisted", payload={"count": len(tasks)}) # [/DEF:persist_tasks:Function] @@ -262,6 +271,7 @@ class TaskPersistenceService: self, limit: int = 100, status: Optional[TaskStatus] = None ) -> List[Task]: with belief_scope("TaskPersistenceService.load_tasks"): + log.reason("Loading tasks from database", payload={"limit": limit, "status": str(status) if status else None}) session: Session = TasksSessionLocal() try: query = session.query(TaskRecord) @@ -305,8 +315,10 @@ class TaskPersistenceService: ) loaded_tasks.append(task) except Exception as e: + log.explore("Failed to reconstruct task from record", error=str(e), payload={"record_id": record.id}) logger.error(f"Failed to reconstruct task {record.id}: {e}") + log.reflect("Tasks loaded from database", payload={"count": len(loaded_tasks)}) return loaded_tasks finally: session.close() @@ -325,14 +337,17 @@ class TaskPersistenceService: if not task_ids: return with belief_scope("TaskPersistenceService.delete_tasks"): + log.reason("Deleting tasks from database", payload={"count": len(task_ids)}) session: Session = TasksSessionLocal() try: session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete( synchronize_session=False ) session.commit() + log.reflect("Tasks deleted from database", payload={"count": len(task_ids)}) except Exception as e: session.rollback() + log.explore("Failed to delete tasks", error=str(e)) logger.error(f"Failed to delete tasks: {e}") finally: session.close() @@ -399,23 +414,26 @@ class TaskLogPersistenceService: if not logs: return with belief_scope("TaskLogPersistenceService.add_logs", f"task_id={task_id}"): + log_pl.reason("Batch inserting log entries", payload={"task_id": task_id, "count": len(logs)}) session: Session = TasksSessionLocal() try: - for log in logs: + for log_entry in logs: record = TaskLogRecord( task_id=task_id, - timestamp=log.timestamp, - level=log.level, - source=log.source or "system", - message=log.message, - metadata_json=json.dumps(log.metadata) - if log.metadata + timestamp=log_entry.timestamp, + level=log_entry.level, + source=log_entry.source or "system", + message=log_entry.message, + metadata_json=json.dumps(log_entry.metadata) + if log_entry.metadata else None, ) session.add(record) session.commit() + log_pl.reflect("Log entries persisted", payload={"task_id": task_id, "count": len(logs)}) except Exception as e: session.rollback() + log_pl.explore("Failed to persist log entries", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to add logs for task {task_id}: {e}") finally: session.close() @@ -436,6 +454,7 @@ class TaskLogPersistenceService: # @RELATION: [DEPENDS_ON] ->[TaskLog] def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]: with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"): + log_pl.reason("Querying task logs", payload={"task_id": task_id, "filter_level": log_filter.level}) session: Session = TasksSessionLocal() try: query = session.query(TaskLogRecord).filter( @@ -480,6 +499,7 @@ class TaskLogPersistenceService: ) ) + log_pl.reflect("Task logs retrieved", payload={"task_id": task_id, "count": len(logs)}) return logs finally: session.close() @@ -500,6 +520,7 @@ class TaskLogPersistenceService: with belief_scope( "TaskLogPersistenceService.get_log_stats", f"task_id={task_id}" ): + log_pl.reason("Aggregating log statistics", payload={"task_id": task_id}) session: Session = TasksSessionLocal() try: # Get total count @@ -531,6 +552,7 @@ class TaskLogPersistenceService: by_source = {source: count for source, count in source_counts} + log_pl.reflect("Log statistics computed", payload={"task_id": task_id, "total": total_count}) return LogStats( total_count=total_count, by_level=by_level, by_source=by_source ) @@ -552,6 +574,7 @@ class TaskLogPersistenceService: with belief_scope( "TaskLogPersistenceService.get_sources", f"task_id={task_id}" ): + log_pl.reason("Querying distinct log sources", payload={"task_id": task_id}) session: Session = TasksSessionLocal() try: from sqlalchemy import distinct @@ -561,7 +584,9 @@ class TaskLogPersistenceService: .filter(TaskLogRecord.task_id == task_id) .all() ) - return [s[0] for s in sources] + result = [s[0] for s in sources] + log_pl.reflect("Log sources retrieved", payload={"task_id": task_id, "sources": result}) + return result finally: session.close() @@ -579,14 +604,17 @@ class TaskLogPersistenceService: with belief_scope( "TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}" ): + log_pl.reason("Deleting logs for task", payload={"task_id": task_id}) session: Session = TasksSessionLocal() try: session.query(TaskLogRecord).filter( TaskLogRecord.task_id == task_id ).delete(synchronize_session=False) session.commit() + log_pl.reflect("Task logs deleted", payload={"task_id": task_id}) except Exception as e: session.rollback() + log_pl.explore("Failed to delete task logs", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to delete logs for task {task_id}: {e}") finally: session.close() @@ -605,14 +633,17 @@ class TaskLogPersistenceService: if not task_ids: return with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"): + log_pl.reason("Deleting logs for multiple tasks", payload={"count": len(task_ids)}) session: Session = TasksSessionLocal() try: session.query(TaskLogRecord).filter( TaskLogRecord.task_id.in_(task_ids) ).delete(synchronize_session=False) session.commit() + log_pl.reflect("Logs deleted for multiple tasks", payload={"count": len(task_ids)}) except Exception as e: session.rollback() + log_pl.explore("Failed to delete logs for tasks", error=str(e)) logger.error(f"Failed to delete logs for tasks: {e}") finally: session.close() diff --git a/backend/src/core/utils/async_network.py b/backend/src/core/utils/async_network.py index 85f2de67..78de1591 100644 --- a/backend/src/core/utils/async_network.py +++ b/backend/src/core/utils/async_network.py @@ -18,6 +18,7 @@ import asyncio import httpx from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger from .network import ( AuthenticationError, DashboardNotFoundError, @@ -28,6 +29,7 @@ from .network import ( ) # [/SECTION] +log = MarkerLogger("AsyncNetwork") # [DEF:AsyncAPIClient:Class] # @COMPLEXITY: 3 @@ -133,10 +135,10 @@ class AsyncAPIClient: self._tokens["csrf_token"] = csrf_response.json()["result"] SupersetAuthCache.set(self._auth_cache_key, self._tokens) self._authenticated = True - app_logger.info("[async_authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s", self.base_url) + log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url}) return self._tokens except Exception as csrf_err: - app_logger.warning("[async_authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s", csrf_err) + log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err)) SupersetAuthCache.invalidate(self._auth_cache_key) self._tokens = {} self._authenticated = False @@ -147,11 +149,11 @@ class AsyncAPIClient: if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): self._tokens = cached_tokens self._authenticated = True - app_logger.info("[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s", self.base_url) + log.reason("Reusing cached Superset auth tokens (after lock wait)", payload={"base_url": self.base_url}) return self._tokens with belief_scope("AsyncAPIClient.authenticate"): - app_logger.info("[async_authenticate][Enter] Authenticating to %s", self.base_url) + log.reason("Performing full authentication", payload={"base_url": self.base_url}) try: login_url = f"{self.api_base_url}/security/login" response = await self._client.post(login_url, json=self.auth) @@ -171,7 +173,7 @@ class AsyncAPIClient: } self._authenticated = True SupersetAuthCache.set(self._auth_cache_key, self._tokens) - app_logger.info("[async_authenticate][Exit] Authenticated successfully.") + log.reflect("Authenticated successfully") return self._tokens except httpx.HTTPStatusError as exc: SupersetAuthCache.invalidate(self._auth_cache_key) diff --git a/backend/src/core/utils/dataset_mapper.py b/backend/src/core/utils/dataset_mapper.py index 59e6dd04..04801c26 100644 --- a/backend/src/core/utils/dataset_mapper.py +++ b/backend/src/core/utils/dataset_mapper.py @@ -12,9 +12,12 @@ import pandas as pd # type: ignore import psycopg2 # type: ignore from typing import Dict, Optional, Any -from ..logger import logger as app_logger, belief_scope +from ..logger import belief_scope +from ..cot_logger import MarkerLogger # [/SECTION] +log = MarkerLogger("DatasetMapper") + # [DEF:DatasetMapper:Class] # @PURPOSE: Класс для меппинга и обновления verbose_map в датасетах Superset. class DatasetMapper: @@ -37,7 +40,7 @@ class DatasetMapper: # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам. def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]: with belief_scope("Fetch comments from PostgreSQL"): - app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name) + log.reason("Fetching comments from PostgreSQL", payload={"table_schema": table_schema, "table_name": table_name}) query = f""" SELECT cols.column_name, @@ -83,9 +86,9 @@ class DatasetMapper: for row in cursor.fetchall(): if row[1]: comments[row[0]] = row[1] - app_logger.info("[get_postgres_comments][Success] Fetched %d comments.", len(comments)) + log.reflect("PostgreSQL comments fetched", payload={"count": len(comments)}) except Exception as e: - app_logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True) + log.explore("Failed to fetch PostgreSQL comments", payload={"table_schema": table_schema, "table_name": table_name}, error=str(e)) raise return comments # [/DEF:get_postgres_comments:Function] @@ -99,14 +102,14 @@ class DatasetMapper: # @RETURN: Dict[str, str] - Словарь с меппингами. def load_excel_mappings(self, file_path: str) -> Dict[str, str]: with belief_scope("Load mappings from Excel"): - app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path) + log.reason("Loading mappings from Excel", payload={"file_path": file_path}) try: df = pd.read_excel(file_path) mappings = df.set_index('column_name')['verbose_name'].to_dict() - app_logger.info("[load_excel_mappings][Success] Loaded %d mappings.", len(mappings)) + log.reflect("Excel mappings loaded", payload={"count": len(mappings)}) return mappings except Exception as e: - app_logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True) + log.explore("Failed to load Excel mappings", payload={"file_path": file_path}, error=str(e)) raise # [/DEF:load_excel_mappings:Function] @@ -128,18 +131,20 @@ class DatasetMapper: # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL. def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None): with belief_scope(f"Run dataset mapping for ID {dataset_id}"): - app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source) + log.reason("Starting dataset mapping", payload={"dataset_id": dataset_id, "source": source}) mappings: Dict[str, str] = {} try: if source in ['postgres', 'both']: - assert postgres_config and table_name and table_schema, "Postgres config is required." + if not (postgres_config and table_name and table_schema): + raise ValueError("Postgres config is required.") mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema)) if source in ['excel', 'both']: - assert excel_path, "Excel path is required." + if not excel_path: + raise ValueError("Excel path is required.") mappings.update(self.load_excel_mappings(excel_path)) if source not in ['postgres', 'excel', 'both']: - app_logger.error("[run_mapping][Failure] Invalid source: %s.", source) + log.explore("Invalid mapping source", payload={"source": source}, error=f"Invalid source: {source}") return dataset_response = superset_client.get_dataset(dataset_id) @@ -224,12 +229,12 @@ class DatasetMapper: payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None} superset_client.update_dataset(dataset_id, payload_for_update) - app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id) + log.reflect("Dataset verbose_name updated", payload={"dataset_id": dataset_id}) else: - app_logger.info("[run_mapping][State] No changes in columns' verbose_name, skipping update.") + log.reflect("No changes in verbose_name, skipping update", payload={"dataset_id": dataset_id}) - except (AssertionError, FileNotFoundError, Exception) as e: - app_logger.error("[run_mapping][Failure] %s", e, exc_info=True) + except (FileNotFoundError, Exception) as e: + log.explore("Dataset mapping failed", payload={"dataset_id": dataset_id, "source": source}, error=str(e)) return # [/DEF:run_mapping:Function] # [/DEF:DatasetMapper:Class] diff --git a/backend/src/core/utils/fileio.py b/backend/src/core/utils/fileio.py index 2f986cb1..8dd5f98e 100644 --- a/backend/src/core/utils/fileio.py +++ b/backend/src/core/utils/fileio.py @@ -20,8 +20,11 @@ import shutil import zlib from dataclasses import dataclass from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger # [/SECTION] +log = MarkerLogger("FileIO") + # [DEF:InvalidZipFormatError:Class] # @PURPOSE: Exception raised when a file is not a valid ZIP archive. class InvalidZipFormatError(Exception): @@ -46,7 +49,7 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode if is_dir: with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir: resource_path = Path(temp_dir) - app_logger.debug("[create_temp_file][State] Created temporary directory: %s", resource_path) + log.reason("Created temporary directory", payload={"path": str(resource_path)}) yield resource_path else: fd, temp_path_str = tempfile.mkstemp(suffix=suffix) @@ -54,19 +57,19 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode os.close(fd) if content: resource_path.write_bytes(content) - app_logger.debug("[create_temp_file][State] Created temporary file: %s", resource_path) + log.reason("Created temporary file", payload={"path": str(resource_path)}) yield resource_path finally: if resource_path and resource_path.exists() and not dry_run: try: if resource_path.is_dir(): shutil.rmtree(resource_path) - app_logger.debug("[create_temp_file][Cleanup] Removed temporary directory: %s", resource_path) + log.reflect("Cleaned up temporary directory", payload={"path": str(resource_path)}) else: resource_path.unlink() - app_logger.debug("[create_temp_file][Cleanup] Removed temporary file: %s", resource_path) + log.reflect("Cleaned up temporary file", payload={"path": str(resource_path)}) except OSError as e: - app_logger.error("[create_temp_file][Failure] Error during cleanup of %s: %s", resource_path, e) + log.explore("Error during temp resource cleanup", payload={"path": str(resource_path)}, error=str(e)) # [/DEF:create_temp_file:Function] # [DEF:remove_empty_directories:Function] @@ -77,20 +80,19 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode # @RETURN: int - Количество удаленных директорий. def remove_empty_directories(root_dir: str) -> int: with belief_scope(f"Remove empty directories in {root_dir}"): - app_logger.info("[remove_empty_directories][Enter] Starting cleanup of empty directories in %s", root_dir) removed_count = 0 if not os.path.isdir(root_dir): - app_logger.error("[remove_empty_directories][Failure] Directory not found: %s", root_dir) + log.explore("Empty directory cleanup skipped - directory not found", payload={"root_dir": root_dir}, error=f"Directory not found: {root_dir}") return 0 for current_dir, _, _ in os.walk(root_dir, topdown=False): if not os.listdir(current_dir): try: os.rmdir(current_dir) removed_count += 1 - app_logger.info("[remove_empty_directories][State] Removed empty directory: %s", current_dir) + log.reason("Removed empty directory", payload={"directory": current_dir}) except OSError as e: - app_logger.error("[remove_empty_directories][Failure] Failed to remove %s: %s", current_dir, e) - app_logger.info("[remove_empty_directories][Exit] Removed %d empty directories.", removed_count) + log.explore("Failed to remove empty directory", payload={"directory": current_dir}, error=str(e)) + log.reflect("Empty directory cleanup complete", payload={"removed_count": removed_count}) return removed_count # [/DEF:remove_empty_directories:Function] @@ -105,10 +107,10 @@ def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]: with belief_scope(f"Read dashboard from {file_path}"): path = Path(file_path) assert path.is_file(), f"Файл дашборда не найден: {file_path}" - app_logger.info("[read_dashboard_from_disk][Enter] Reading file: %s", file_path) + log.reason("Reading dashboard file from disk", payload={"file_path": file_path}) content = path.read_bytes() if not content: - app_logger.warning("[read_dashboard_from_disk][Warning] File is empty: %s", file_path) + log.explore("Dashboard file is empty", payload={"file_path": file_path}, error="Empty dashboard file") return content, path.name # [/DEF:read_dashboard_from_disk:Function] @@ -150,20 +152,20 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool with belief_scope(f"Archive exports in {output_dir}"): output_path = Path(output_dir) if not output_path.is_dir(): - app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir) + log.explore("Archive directory not found", payload={"output_dir": output_dir}, error=f"Archive directory not found: {output_dir}") return - app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir) + log.reason("Managing archive exports", payload={"output_dir": output_dir}) # 1. Collect all zip files zip_files = list(output_path.glob("*.zip")) if not zip_files: - app_logger.info("[archive_exports][State] No zip files found in %s", output_dir) + log.reason("No zip files found in archive directory", payload={"output_dir": output_dir}) return # 2. Deduplication if deduplicate: - app_logger.info("[archive_exports][State] Starting deduplication...") + log.reason("Starting archive deduplication") checksums = {} files_to_remove = [] @@ -175,19 +177,19 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool crc = calculate_crc32(file_path) if crc in checksums: files_to_remove.append(file_path) - app_logger.debug("[archive_exports][State] Duplicate found: %s (same as %s)", file_path.name, checksums[crc].name) + log.reason("Duplicate archive found (same checksum)", payload={"file": file_path.name, "original": checksums[crc].name}) else: checksums[crc] = file_path except Exception as e: - app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e) + log.explore("Failed to calculate CRC32 for archive", payload={"file": str(file_path)}, error=str(e)) for f in files_to_remove: try: f.unlink() zip_files.remove(f) - app_logger.info("[archive_exports][State] Removed duplicate: %s", f.name) + log.reason("Removed duplicate archive", payload={"file": f.name}) except OSError as e: - app_logger.error("[archive_exports][Failure] Failed to remove duplicate %s: %s", f, e) + log.explore("Failed to remove duplicate archive", payload={"file": str(f)}, error=str(e)) # 3. Retention Policy files_with_dates = [] @@ -215,9 +217,9 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool if file_path not in files_to_keep: try: file_path.unlink() - app_logger.info("[archive_exports][State] Removed by retention policy: %s", file_path.name) + log.reason("Removed archive by retention policy", payload={"file": file_path.name}) except OSError as e: - app_logger.error("[archive_exports][Failure] Failed to remove %s: %s", file_path, e) + log.explore("Failed to remove archive by retention policy", payload={"file": str(file_path)}, error=str(e)) # [/DEF:archive_exports:Function] # [DEF:apply_retention_policy:Function] @@ -251,7 +253,7 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re files_to_keep.update(daily_files) files_to_keep.update(weekly_files[:policy.weekly]) files_to_keep.update(monthly_files[:policy.monthly]) - app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep)) + log.reflect("Retention policy applied", payload={"files_to_keep": len(files_to_keep)}) return files_to_keep # [/DEF:apply_retention_policy:Function] @@ -267,22 +269,22 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re # @THROW: InvalidZipFormatError - При ошибке формата ZIP. def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]: with belief_scope("Save and unpack dashboard"): - app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack) + log.reason("Processing dashboard ZIP save", payload={"unpack": unpack}) try: output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) zip_name = sanitize_filename(original_filename) if original_filename else f"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip" zip_path = output_path / zip_name zip_path.write_bytes(zip_content) - app_logger.info("[save_and_unpack_dashboard][State] Dashboard saved to: %s", zip_path) + log.reason("Dashboard ZIP saved to disk", payload={"zip_path": str(zip_path)}) if unpack: with zipfile.ZipFile(zip_path, 'r') as zip_ref: zip_ref.extractall(output_path) - app_logger.info("[save_and_unpack_dashboard][State] Dashboard unpacked to: %s", output_path) + log.reason("Dashboard unpacked to directory", payload={"output_path": str(output_path)}) return zip_path, output_path return zip_path, None except zipfile.BadZipFile as e: - app_logger.error("[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s", e) + log.explore("Invalid ZIP archive format", payload={"output_dir": str(output_dir)}, error=str(e)) raise InvalidZipFormatError(f"Invalid ZIP file: {e}") from e # [/DEF:save_and_unpack_dashboard:Function] @@ -298,7 +300,7 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], # @PARAM: replace_string (Optional[LiteralString]) - Строка для замены. def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None: with belief_scope("Update YAML configurations"): - app_logger.info("[update_yamls][Enter] Starting YAML configuration update.") + log.reason("Starting YAML configuration update", payload={"path": path}) dir_path = Path(path) assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией" @@ -323,7 +325,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ with open(file_path, 'r', encoding='utf-8') as f: content = f.read() except Exception as e: - app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e) + log.explore("Failed to read YAML file", payload={"file_path": str(file_path)}, error=str(e)) return # Если задан pattern и replace_string, применяем замену по регулярному выражению if regexp_pattern and replace_string: @@ -332,9 +334,9 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ if new_content != content: with open(file_path, 'w', encoding='utf-8') as f: f.write(new_content) - app_logger.info("[_update_yaml_file][State] Updated %s using regex pattern", file_path) + log.reason("Updated YAML file using regex pattern", payload={"file_path": str(file_path)}) except Exception as e: - app_logger.error("[_update_yaml_file][Failure] Error applying regex to %s: %s", file_path, e) + log.explore("Error applying regex to YAML file", payload={"file_path": str(file_path)}, error=str(e)) # Если заданы конфигурации, заменяем значения (поддержка old/new) if db_configs: try: @@ -367,12 +369,12 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ # [/DEF:replacer:Function] modified_content = re.sub(pattern, replacer, modified_content) - app_logger.info("[_update_yaml_file][State] Replaced '%s' with '%s' for key %s in %s", old_val, new_val, key, file_path) + log.reason("Replaced YAML config value", payload={"key": key, "file_path": str(file_path)}) # Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование with open(file_path, 'w', encoding='utf-8') as f: f.write(modified_content) except Exception as e: - app_logger.error("[_update_yaml_file][Failure] Error performing raw replacement in %s: %s", file_path, e) + log.explore("Error performing raw replacement in YAML", payload={"file_path": str(file_path)}, error=str(e)) # [/DEF:_update_yaml_file:Function] # [DEF:create_dashboard_export:Function] @@ -385,7 +387,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ # @RETURN: bool - `True` при успехе, `False` при ошибке. def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool: with belief_scope(f"Create dashboard export: {zip_path}"): - app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path) + log.reason("Packing dashboard export", payload={"source_paths": [str(p) for p in source_paths], "zip_path": str(zip_path)}) try: exclude_ext = [ext.lower() for ext in exclude_extensions or []] with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: @@ -396,10 +398,10 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union if item.is_file() and item.suffix.lower() not in exclude_ext: arcname = item.relative_to(src_path.parent) zipf.write(item, arcname) - app_logger.info("[create_dashboard_export][Exit] Archive created: %s", zip_path) + log.reflect("Dashboard archive created", payload={"zip_path": str(zip_path)}) return True except (IOError, zipfile.BadZipFile, AssertionError) as e: - app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True) + log.explore("Dashboard archive creation failed", payload={"zip_path": str(zip_path)}, error=str(e)) return False # [/DEF:create_dashboard_export:Function] @@ -439,7 +441,7 @@ def consolidate_archive_folders(root_directory: Path) -> None: assert isinstance(root_directory, Path), "root_directory must be a Path object." assert root_directory.is_dir(), "root_directory must be an existing directory." - app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory) + log.reason("Consolidating archive folders", payload={"root_directory": str(root_directory)}) # Собираем все директории с архивами archive_dirs = [] for item in root_directory.iterdir(): @@ -462,7 +464,7 @@ def consolidate_archive_folders(root_directory: Path) -> None: # Создаем целевую директорию target_dir = root_directory / slug target_dir.mkdir(exist_ok=True) - app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir) + log.reason("Consolidating archive directories", payload={"slug": slug, "dir_count": len(dirs), "target": str(target_dir)}) # Перемещаем содержимое for source_dir in dirs: if source_dir == target_dir: @@ -475,13 +477,13 @@ def consolidate_archive_folders(root_directory: Path) -> None: else: shutil.move(str(item), str(dest_item)) except Exception as e: - app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e) + log.explore("Failed to move item during consolidation", payload={"source": str(item), "dest": str(dest_item)}, error=str(e)) # Удаляем исходную директорию try: source_dir.rmdir() - app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir) + log.reason("Removed source directory after consolidation", payload={"directory": str(source_dir)}) except Exception as e: - app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e) + log.explore("Failed to remove source directory after consolidation", payload={"directory": str(source_dir)}, error=str(e)) # [/DEF:consolidate_archive_folders:Function] # [/DEF:FileIO:Module] \ No newline at end of file diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index ff5d70f2..97d5154e 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -19,8 +19,11 @@ from requests.adapters import HTTPAdapter import urllib3 from urllib3.util.retry import Retry from ..logger import logger as app_logger, belief_scope +from ..cot_logger import MarkerLogger # [/SECTION] +log = MarkerLogger("Network") + # [DEF:SupersetAPIError:Class] # @COMPLEXITY: 1 # @PURPOSE: Base exception for all Superset API related errors. @@ -169,7 +172,7 @@ class APIClient: # @POST: APIClient instance is initialized with a session. def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): with belief_scope("__init__"): - app_logger.info("[APIClient.__init__][Entry] Initializing APIClient.") + log.reason("Initializing APIClient") self.base_url: str = self._normalize_base_url(config.get("base_url", "")) self.api_base_url: str = f"{self.base_url}/api/v1" self.auth = config.get("auth") @@ -182,7 +185,7 @@ class APIClient: verify_ssl, ) self._authenticated = False - app_logger.info("[APIClient.__init__][Exit] APIClient initialized.") + log.reflect("APIClient initialized") # [/DEF:APIClient.__init__:Function] # [DEF:_init_session:Function] @@ -222,7 +225,7 @@ class APIClient: if not self.request_settings["verify_ssl"]: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - app_logger.warning("[_init_session][State] SSL verification disabled.") + log.reflect("SSL verification disabled for session", payload={"base_url": self.base_url}) # When verify_ssl is false, we should also disable hostname verification session.verify = False else: @@ -277,7 +280,7 @@ class APIClient: # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances. def authenticate(self) -> Dict[str, str]: with belief_scope("authenticate"): - app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url) + log.reason("Authenticating to Superset", payload={"base_url": self.base_url}) cached_tokens = SupersetAuthCache.get(self._auth_cache_key) if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): @@ -295,10 +298,10 @@ class APIClient: # Update CSRF token (may have rotated) self._tokens["csrf_token"] = csrf_response.json()["result"] SupersetAuthCache.set(self._auth_cache_key, self._tokens) - app_logger.info("[authenticate][CacheHit+CSRF] Reused tokens + refreshed CSRF for %s", self.base_url) + log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url}) except Exception as csrf_err: # CSRF refresh failed — likely access_token expired, do full re-auth - app_logger.warning("[authenticate][CacheMiss] CSRF refresh failed, performing full re-auth: %s", csrf_err) + log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err)) SupersetAuthCache.invalidate(self._auth_cache_key) # Fall through to full login below self._tokens = {} @@ -309,13 +312,12 @@ class APIClient: login_url = f"{self.api_base_url}/security/login" # Log the payload keys and values (masking password) masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()} - app_logger.info(f"[authenticate][Debug] Login URL: {login_url}") - app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}") + log.reason("Performing full Superset login", payload={"login_url": login_url, "auth_payload": masked_auth}) response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"]) if response.status_code != 200: - app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}") + log.explore("Superset login returned non-200 status", payload={"status_code": response.status_code, "response": response.text}, error=f"HTTP {response.status_code}: {response.text[:200]}") response.raise_for_status() access_token = response.json()["access_token"] @@ -327,7 +329,7 @@ class APIClient: self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]} self._authenticated = True SupersetAuthCache.set(self._auth_cache_key, self._tokens) - app_logger.info("[authenticate][Exit] Authenticated successfully.") + log.reflect("Authenticated successfully") return self._tokens except requests.exceptions.HTTPError as e: SupersetAuthCache.invalidate(self._auth_cache_key) @@ -340,7 +342,7 @@ class APIClient: raise NetworkError(f"Network or parsing error during authentication: {e}") from e self._authenticated = True - app_logger.info("[authenticate][Exit] Authenticated successfully (from cache).") + log.reflect("Authenticated successfully (from cached tokens)") return self._tokens # [/DEF:APIClient.authenticate:Function] @@ -502,7 +504,7 @@ class APIClient: try: return response.json() except Exception as json_e: - app_logger.debug(f"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...") + log.explore("Upload response not valid JSON", payload={"preview": response.text[:200]}, error=str(json_e)) raise SupersetAPIError(f"API error during upload: Response is not valid JSON: {json_e}") from json_e return response.json() except requests.exceptions.HTTPError as e: @@ -554,7 +556,7 @@ class APIClient: if total_count is None: total_count = response_json.get(count_field, len(first_page_results)) - app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}") + log.reason("Total count resolved from first page", payload={"total_count": total_count}) # Fetch remaining pages total_pages = (total_count + page_size - 1) // page_size diff --git a/backend/src/core/ws_log_handler.py b/backend/src/core/ws_log_handler.py new file mode 100644 index 00000000..599df4c4 --- /dev/null +++ b/backend/src/core/ws_log_handler.py @@ -0,0 +1,70 @@ +# #region WsLogHandlerModule [C:3] [TYPE Module] [SEMANTICS logging,handler,websocket,buffer] +# @BRIEF WebSocket log handler module providing LogEntry DTO and WebSocketLogHandler for buffered log streaming. +# @RELATION DEPENDS_ON -> [LogEntry] +# @RELATION COMPOSES -> [CotJsonFormatter] +import logging +from typing import Dict, Any, List, Optional +from collections import deque +from datetime import datetime + +from pydantic import BaseModel, Field + + +# #region LogEntry [C:1] [TYPE Class] [SEMANTICS log,entry,record,pydantic] +class LogEntry(BaseModel): + timestamp: datetime = Field(default_factory=datetime.utcnow) + level: str + message: str + context: Optional[Dict[str, Any]] = None +# #endregion LogEntry + + +# #region WebSocketLogHandler [C:3] [TYPE Class] [SEMANTICS logging,handler,websocket,buffer] +# @BRIEF Custom logging handler that captures log records into a fixed-capacity buffer for WebSocket streaming. +# @RELATION DEPENDS_ON -> [LogEntry] +# @RELATION COMPOSES -> [CotJsonFormatter] +class WebSocketLogHandler(logging.Handler): + """ + A logging handler that stores log records and can be extended to send them + over WebSockets. + """ + + # #region WebSocketLogHandler.__init__ [C:1] [TYPE Function] [SEMANTICS init,handler,buffer] + def __init__(self, capacity: int = 1000): + super().__init__() + self.log_buffer: deque[LogEntry] = deque(maxlen=capacity) + # #endregion WebSocketLogHandler.__init__ + + # #region WebSocketLogHandler.emit [C:2] [TYPE Function] [SEMANTICS log,capture,format,buffer] + # @BRIEF Captures a log record, formats it, and stores it in the buffer as a LogEntry. + def emit(self, record: logging.LogRecord): + try: + log_entry = LogEntry( + level=record.levelname, + message=self.format(record), + context={ + "name": record.name, + "pathname": record.pathname, + "lineno": record.lineno, + "funcName": record.funcName, + "process": record.process, + "thread": record.thread, + } + ) + self.log_buffer.append(log_entry) + except Exception: + self.handleError(record) + # #endregion WebSocketLogHandler.emit + + # #region WebSocketLogHandler.get_recent_logs [C:2] [TYPE Function] [SEMANTICS log,retrieve,buffer] + # @BRIEF Returns a list of recent log entries from the buffer. + def get_recent_logs(self) -> List[LogEntry]: + """ + Returns a list of recent log entries from the buffer. + """ + return list(self.log_buffer) + # #endregion WebSocketLogHandler.get_recent_logs + + +# #endregion WebSocketLogHandler +# #endregion WsLogHandlerModule diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index 4e78dd1e..a49e11d2 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -41,11 +41,13 @@ from .services.clean_release.repository import CleanReleaseRepository from .services.clean_release.facade import CleanReleaseFacade from .services.reports.report_service import ReportsService from .core.database import init_db, get_auth_db, get_db -from .core.logger import logger +from .core.cot_logger import MarkerLogger from .core.auth.jwt import decode_token from .core.auth.repository import AuthRepository from .models.auth import User +log = MarkerLogger("Dependencies") + # Initialize singletons lazily to avoid import-time DB side effects during test collection. # Use absolute path relative to this file to ensure plugins are found regardless of CWD project_root = Path(__file__).parent.parent.parent @@ -94,8 +96,8 @@ def get_plugin_loader() -> PluginLoader: global plugin_loader if plugin_loader is None: plugin_loader = PluginLoader(plugin_dir=str(plugin_dir)) - logger.info(f"PluginLoader initialized with directory: {plugin_dir}") - logger.info( + log.reason(f"PluginLoader initialized with directory: {plugin_dir}") + log.reason( f"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}" ) return plugin_loader @@ -115,7 +117,7 @@ def get_task_manager() -> TaskManager: global task_manager if task_manager is None: task_manager = TaskManager(get_plugin_loader()) - logger.info("TaskManager initialized") + log.reason("TaskManager initialized") return task_manager @@ -133,7 +135,7 @@ def get_scheduler_service() -> SchedulerService: global scheduler_service if scheduler_service is None: scheduler_service = SchedulerService(get_task_manager(), get_config_manager()) - logger.info("SchedulerService initialized") + log.reason("SchedulerService initialized") return scheduler_service @@ -151,7 +153,7 @@ def get_resource_service() -> ResourceService: global resource_service if resource_service is None: resource_service = ResourceService() - logger.info("ResourceService initialized") + log.reason("ResourceService initialized") return resource_service diff --git a/backend/src/plugins/git_plugin.py b/backend/src/plugins/git_plugin.py index 38ed66d7..1b69a931 100644 --- a/backend/src/plugins/git_plugin.py +++ b/backend/src/plugins/git_plugin.py @@ -22,6 +22,9 @@ from typing import Dict, Any, Optional from src.core.plugin_base import PluginBase from src.services.git_service import GitService from src.core.logger import logger as app_logger, belief_scope +from src.core.cot_logger import MarkerLogger + +log = MarkerLogger("GitPlugin") from src.core.config_manager import ConfigManager from src.core.superset_client import SupersetClient from src.core.task_manager.context import TaskContext @@ -39,7 +42,7 @@ class GitPlugin(PluginBase): # @POST: Инициализированы git_service и config_manager. def __init__(self): with belief_scope("GitPlugin.__init__"): - app_logger.info("Initializing GitPlugin.") + log.reason("Initializing GitPlugin.") self.git_service = GitService() # Robust config path resolution: @@ -54,13 +57,13 @@ class GitPlugin(PluginBase): try: from src.dependencies import config_manager self.config_manager = config_manager - app_logger.info("GitPlugin initialized using shared config_manager.") + log.reflect("GitPlugin initialized using shared config_manager.") return except Exception: config_path = "config.json" self.config_manager = ConfigManager(config_path) - app_logger.info(f"GitPlugin initialized with {config_path}") + log.reflect(f"GitPlugin initialized with {config_path}") # [/DEF:__init__:Function] @property @@ -137,7 +140,7 @@ class GitPlugin(PluginBase): # @POST: Плагин готов к выполнению задач. async def initialize(self): with belief_scope("GitPlugin.initialize"): - app_logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.") + log.reason("Initializing Git Integration Plugin logic") # [DEF:execute:Function] # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext. @@ -248,15 +251,15 @@ class GitPlugin(PluginBase): # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff) try: repo.git.add(A=True) - app_logger.info("[_handle_sync][Action] Changes staged in git") + log.reason("Changes staged in git") except Exception as ge: - app_logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}") + log.explore("Failed to stage changes", error=str(ge)) - app_logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.") + log.reflect("Dashboard synced successfully", payload={"dashboard_id": dashboard_id}) return {"status": "success", "message": "Dashboard synced and flattened in local repository"} except Exception as e: - app_logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}") + log.explore("Sync failed", error=str(e)) raise # [/DEF:_handle_sync:Function] @@ -318,16 +321,16 @@ class GitPlugin(PluginBase): f.write(zip_buffer.getvalue()) try: - app_logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}") + log.reason("Importing dashboard", payload={"environment": env.name}) result = client.import_dashboard(temp_zip_path) - app_logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.") + log.reflect("Deployment successful", payload={"dashboard_id": dashboard_id}) return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result} finally: if temp_zip_path.exists(): os.remove(temp_zip_path) except Exception as e: - app_logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}") + log.explore("Deployment failed", error=str(e)) raise # [/DEF:_handle_deploy:Function] @@ -339,13 +342,13 @@ class GitPlugin(PluginBase): # @RETURN: Environment - Объект конфигурации окружения. def _get_env(self, env_id: Optional[str] = None): with belief_scope("GitPlugin._get_env"): - app_logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}") + log.reason("Fetching environment", payload={"env_id": env_id}) # Priority 1: ConfigManager (config.json) if env_id: env = self.config_manager.get_environment(env_id) if env: - app_logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}") + log.reflect("Found environment by ID in ConfigManager", payload={"name": env.name}) return env # Priority 2: Database (DeploymentEnvironment) @@ -363,7 +366,7 @@ class GitPlugin(PluginBase): db_env = db.query(DeploymentEnvironment).first() if db_env: - app_logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}") + log.reflect("Found environment in DB", payload={"name": db_env.name}) from src.core.config_models import Environment # Use token as password for SupersetClient return Environment( @@ -385,14 +388,14 @@ class GitPlugin(PluginBase): # but we have other envs, maybe it's one of them? env = next((e for e in envs if e.id == env_id), None) if env: - app_logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list") + log.reflect("Found environment in ConfigManager list", payload={"env_id": env_id}) return env if not env_id: - app_logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}") + log.reflect("Using first environment from ConfigManager", payload={"name": envs[0].name}) return envs[0] - app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}") + log.explore("No environments configured", payload={"env_id": env_id}, error="No Superset environments found in config or database") raise ValueError("No environments configured. Please add a Superset Environment in Settings.") # [/DEF:_get_env:Function] diff --git a/backend/src/plugins/migration.py b/backend/src/plugins/migration.py index 400713a9..6bba26f0 100755 --- a/backend/src/plugins/migration.py +++ b/backend/src/plugins/migration.py @@ -18,8 +18,11 @@ from typing import Dict, Any, Optional import re from ..core.plugin_base import PluginBase -from ..core.logger import belief_scope, logger as app_logger +from ..core.logger import belief_scope +from ..core.cot_logger import MarkerLogger from ..core.superset_client import SupersetClient + +log = MarkerLogger("MigrationPlugin") from ..core.utils.fileio import create_temp_file from ..dependencies import get_config_manager from ..core.migration_engine import MigrationEngine @@ -105,7 +108,7 @@ class MigrationPlugin(PluginBase): # @RETURN: Dict[str, Any] def get_schema(self) -> Dict[str, Any]: with belief_scope("MigrationPlugin.get_schema"): - app_logger.reason("Generating migration UI schema") + log.reason("Generating migration UI schema") config_manager = get_config_manager() envs = [e.name for e in config_manager.get_environments()] @@ -148,7 +151,7 @@ class MigrationPlugin(PluginBase): }, "required": ["from_env", "to_env", "dashboard_regex"], } - app_logger.reflect("Schema generated successfully", extra={"environments_count": len(envs)}) + log.reflect("Schema generated successfully", payload={"environments_count": len(envs)}) return schema # [/DEF:get_schema:Function] @@ -168,7 +171,7 @@ class MigrationPlugin(PluginBase): # @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS] async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): with belief_scope("MigrationPlugin.execute"): - app_logger.reason("Evaluating migration task parameters", extra={"params": params}) + log.reason("Evaluating migration task parameters", payload={"params": params}) source_env_id = params.get("source_env_id") target_env_id = params.get("target_env_id") @@ -184,11 +187,11 @@ class MigrationPlugin(PluginBase): from ..dependencies import get_task_manager tm = get_task_manager() - log = context.logger if context else app_logger - superset_log = log.with_source("superset_api") if context else log - migration_log = log.with_source("migration") if context else log + exec_log = context.logger if context else log + superset_log = exec_log.with_source("superset_api") if context else exec_log + migration_log = exec_log.with_source("migration") if context else exec_log - log.info("Starting migration task.") + exec_log.info("Starting migration task.") try: config_manager = get_config_manager() @@ -199,13 +202,13 @@ class MigrationPlugin(PluginBase): tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None) if not src_env or not tgt_env: - app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name}) + log.explore("Environment resolution failed", error="Could not resolve source or target environment", payload={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name}) raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}") from_env_name = src_env.name to_env_name = tgt_env.name - app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name}) + log.reason("Environments resolved successfully", payload={"from": from_env_name, "to": to_env_name}) migration_result = { "status": "SUCCESS", @@ -232,12 +235,12 @@ class MigrationPlugin(PluginBase): regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE) dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))] else: - app_logger.explore("No deterministic selection criteria provided") + log.explore("No deterministic selection criteria provided", error="No dashboard selection criteria (selected_ids or dashboard_regex)") migration_result["status"] = "NO_SELECTION" return migration_result if not dashboards_to_migrate: - app_logger.explore("Zero dashboards match selection criteria") + log.explore("Zero dashboards match selection criteria", error="No dashboards matched the given selection criteria") migration_result["status"] = "NO_MATCHES" return migration_result @@ -249,7 +252,7 @@ class MigrationPlugin(PluginBase): db_mapping = {} if replace_db_config: - app_logger.reason("Fetching environment DB mappings from catalog") + log.reason("Fetching environment DB mappings from catalog") db = SessionLocal() try: src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first() @@ -263,7 +266,7 @@ class MigrationPlugin(PluginBase): stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings} stored_map_dict.update(db_mapping) db_mapping = stored_map_dict - log.info(f"Loaded {len(stored_mappings)} database mappings from database.") + exec_log.info(f"Loaded {len(stored_mappings)} database mappings from database.") finally: db.close() @@ -273,7 +276,7 @@ class MigrationPlugin(PluginBase): # Migration Loop for dash in dashboards_to_migrate: dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"] - app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id}) + log.reason(f"Starting pipeline for dashboard '{title}'", payload={"dash_id": dash_id}) try: exported_content, _ = from_c.export_dashboard(dash_id) @@ -291,10 +294,10 @@ class MigrationPlugin(PluginBase): if not success and replace_db_config: if task_id: - app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id}) + log.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", error="AST transform blocked by missing mappings", payload={"task_id": task_id}) await tm.wait_for_resolution(task_id) - app_logger.reason("Task resumed, re-evaluating mapping states") + log.reason("Task resumed, re-evaluating mapping states") db = SessionLocal() try: src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first() @@ -317,12 +320,12 @@ class MigrationPlugin(PluginBase): ) if success: - app_logger.reason("Pushing transformed ZIP to target Superset") + log.reason("Pushing transformed ZIP to target Superset") to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug) migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) - app_logger.reflect("Import successful", extra={"title": title}) + log.reflect("Import successful", payload={"title": title}) else: - app_logger.explore("Transformation strictly failed, bypassing ingestion") + log.explore("Transformation strictly failed, bypassing ingestion", error="Dashboard ZIP transformation failed") migration_log.error(f"Failed to transform ZIP for dashboard {title}") migration_result["failed_dashboards"].append({ "id": dash_id, "title": title, "error": "Failed to transform ZIP" @@ -340,7 +343,7 @@ class MigrationPlugin(PluginBase): if match_alt: db_name = match_alt.group(1) - app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name}) + log.explore("Missing DB password detected during ingestion. Escalating to UI.", error="Missing DB password for import", payload={"db_name": db_name}) if task_id: tm.await_input(task_id, { @@ -354,15 +357,15 @@ class MigrationPlugin(PluginBase): passwords = task.params.get("passwords", {}) if passwords: - app_logger.reason(f"Retrying import for {title} with injected credentials") + log.reason(f"Retrying import for {title} with injected credentials") to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords) migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) - app_logger.reflect("Password injection unblocked import") + log.reflect("Password injection unblocked import") if "passwords" in task.params: del task.params["passwords"] continue - app_logger.explore(f"Catastrophic dashboard ingestion failure: {exc}") + log.explore("Catastrophic dashboard ingestion failure", error=f"Dashboard ingestion failed: {exc}") migration_result["failed_dashboards"].append({"id": dash_id, "title": title, "error": str(exc)}) if migration_result["failed_dashboards"]: @@ -370,20 +373,20 @@ class MigrationPlugin(PluginBase): # Post-Migration ID Mapping Synchronization try: - app_logger.reason("Executing incremental ID catalog sync on target") + log.reason("Executing incremental ID catalog sync on target") db_session = SessionLocal() mapping_service = IdMappingService(db_session) mapping_service.sync_environment(tgt_env.id, to_c, incremental=True) db_session.close() - app_logger.reflect("Incremental catalog sync closed out cleanly") + log.reflect("Incremental catalog sync closed out cleanly") except Exception as sync_exc: - app_logger.explore(f"ID Mapping sync failed, mapping state might be degraded: {sync_exc}") + log.explore("ID Mapping sync failed, mapping state might be degraded", error=f"ID Mapping sync error: {sync_exc}") - app_logger.reflect("Migration cycle fully resolved", extra={"result": migration_result}) + log.reflect("Migration cycle fully resolved", payload={"result": migration_result}) return migration_result except Exception as e: - app_logger.explore(f"Fatal plugin failure: {e}", exc_info=True) + log.explore("Fatal plugin failure", error=f"Plugin execution failed: {e}", exc_info=True) raise e # [/DEF:execute:Function] # [/DEF:MigrationPlugin:Class] diff --git a/backend/src/plugins/storage/plugin.py b/backend/src/plugins/storage/plugin.py index cc4889d9..cce0e4fa 100644 --- a/backend/src/plugins/storage/plugin.py +++ b/backend/src/plugins/storage/plugin.py @@ -18,8 +18,11 @@ from typing import Dict, Any, List, Optional from fastapi import UploadFile from ...core.plugin_base import PluginBase -from ...core.logger import belief_scope, logger +from ...core.logger import belief_scope +from ...core.cot_logger import MarkerLogger from ...models.storage import StoredFile, FileCategory + +log = MarkerLogger("StoragePlugin") from ...dependencies import get_config_manager from ...core.task_manager.context import TaskContext # [/SECTION] @@ -121,14 +124,7 @@ class StoragePlugin(PluginBase): # @POST: Task is executed and logged. async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): with belief_scope("StoragePlugin:execute"): - # Use TaskContext logger if available, otherwise fall back to app logger - log = context.logger if context else logger - - # Create sub-loggers for different components - storage_log = log.with_source("storage") if context else log - log.with_source("filesystem") if context else log - - storage_log.info(f"Executing with params: {params}") + log.reason(f"Executing with params: {params}") # [/DEF:execute:Function] # [DEF:get_storage_root:Function] @@ -172,7 +168,7 @@ class StoragePlugin(PluginBase): # Clean up any double slashes or leading/trailing slashes for relative path return os.path.normpath(resolved).strip("/") except KeyError as e: - logger.warning(f"[StoragePlugin][Coherence:Failed] Missing variable for path resolution: {e}") + log.explore("Missing variable for path resolution", error=str(e)) # Fallback to literal pattern if formatting fails partially (or handle as needed) return pattern.replace("{", "").replace("}", "") # [/DEF:resolve_path:Function] @@ -189,7 +185,7 @@ class StoragePlugin(PluginBase): # Use singular name for consistency with BackupPlugin and GitService path = root / category.value path.mkdir(parents=True, exist_ok=True) - logger.debug(f"[StoragePlugin][Action] Ensured directory: {path}") + log.reason(f"Ensured directory: {path}") # [/DEF:ensure_directories:Function] # [DEF:validate_path:Function] @@ -203,7 +199,7 @@ class StoragePlugin(PluginBase): try: resolved.relative_to(root) except ValueError: - logger.error(f"[StoragePlugin][Coherence:Failed] Path traversal detected: {resolved} is not under {root}") + log.explore(f"Path traversal detected: {resolved} is not under {root}", error="Path outside storage root") raise ValueError("Access denied: Path is outside of storage root.") return resolved # [/DEF:validate_path:Function] @@ -224,8 +220,8 @@ class StoragePlugin(PluginBase): ) -> List[StoredFile]: with belief_scope("StoragePlugin:list_files"): root = self.get_storage_root() - logger.info( - f"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}" + log.reason( + f"Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}" ) files = [] @@ -261,7 +257,7 @@ class StoragePlugin(PluginBase): if not target_dir.exists(): continue - logger.debug(f"[StoragePlugin][Action] Scanning directory: {target_dir}") + log.reason(f"Scanning directory: {target_dir}") if recursive: for current_root, dirs, filenames in os.walk(target_dir): @@ -361,7 +357,7 @@ class StoragePlugin(PluginBase): shutil.rmtree(full_path) else: full_path.unlink() - logger.info(f"[StoragePlugin][Action] Deleted: {full_path}") + log.reflect(f"Deleted: {full_path}") else: raise FileNotFoundError(f"Item {path} not found") # [/DEF:delete_file:Function] diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index b8b56baf..fc779981 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -17,7 +17,8 @@ import re from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session from sqlalchemy import func -from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger +from ...core.logger import belief_scope from ...models.translate import ( TerminologyDictionary, DictionaryEntry, @@ -26,6 +27,8 @@ from ...models.translate import ( ) from ._utils import _normalize_term, _detect_delimiter +log = MarkerLogger("DictionaryManager") + # #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import] # @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering. @@ -45,7 +48,7 @@ class DictionaryManager: is_active: bool = True, ) -> TerminologyDictionary: with belief_scope("DictionaryManager.create_dictionary"): - logger.reason("Creating dictionary", {"name": name, "source": source_dialect, "target": target_dialect}) + log.reason("Creating dictionary", payload={"name": name, "source": source_dialect, "target": target_dialect}) dictionary = TerminologyDictionary( name=name, description=description, @@ -57,7 +60,7 @@ class DictionaryManager: db.add(dictionary) db.commit() db.refresh(dictionary) - logger.reflect("Dictionary created", {"id": dictionary.id}) + log.reflect("Dictionary created", payload={"id": dictionary.id}) return dictionary # #endregion DictionaryManager.create_dictionary @@ -74,7 +77,7 @@ class DictionaryManager: dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary not found: {dict_id}") - logger.reason("Updating dictionary", {"id": dict_id}) + log.reason("Updating dictionary", payload={"id": dict_id}) if name is not None: dictionary.name = name if description is not None: @@ -87,7 +90,7 @@ class DictionaryManager: dictionary.is_active = is_active db.commit() db.refresh(dictionary) - logger.reflect("Dictionary updated", {"id": dictionary.id}) + log.reflect("Dictionary updated", payload={"id": dictionary.id}) return dictionary # #endregion DictionaryManager.update_dictionary @@ -119,19 +122,19 @@ class DictionaryManager: ) if attached_jobs > 0: - logger.explore("Delete blocked: dictionary attached to active jobs", {"dict_id": dict_id, "jobs": attached_jobs}) + log.explore("Delete blocked: dictionary attached to active jobs", payload={"dict_id": dict_id, "jobs": attached_jobs}, error=f"Dictionary attached to {attached_jobs} active job(s)") raise ValueError( f"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). " "Remove dictionary associations from jobs first." ) - logger.reason("Deleting dictionary", {"id": dict_id}) + log.reason("Deleting dictionary", payload={"id": dict_id}) # Delete entries and job-dictionary links first db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete() db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete() db.delete(dictionary) db.commit() - logger.reflect("Dictionary deleted", {"id": dict_id}) + log.reflect("Dictionary deleted", payload={"id": dict_id}) # #endregion DictionaryManager.delete_dictionary # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get] @@ -185,7 +188,7 @@ class DictionaryManager: f"(id={existing.id}). Use overwrite or keep_existing conflict mode." ) - logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term}) + log.reason("Adding dictionary entry", payload={"dict_id": dict_id, "term": source_term}) entry = DictionaryEntry( dictionary_id=dict_id, source_term=source_term.strip(), @@ -196,7 +199,7 @@ class DictionaryManager: db.add(entry) db.commit() db.refresh(entry) - logger.reflect("Entry added", {"entry_id": entry.id}) + log.reflect("Entry added", payload={"entry_id": entry.id}) return entry # #endregion DictionaryManager.add_entry @@ -213,7 +216,7 @@ class DictionaryManager: if not entry: raise ValueError(f"Entry not found: {entry_id}") - logger.reason("Editing dictionary entry", {"entry_id": entry_id}) + log.reason("Editing dictionary entry", payload={"entry_id": entry_id}) if source_term is not None: normalized = _normalize_term(source_term) # Check uniqueness within the same dictionary @@ -239,7 +242,7 @@ class DictionaryManager: entry.context_notes = context_notes.strip() if context_notes else None db.commit() db.refresh(entry) - logger.reflect("Entry updated", {"entry_id": entry.id}) + log.reflect("Entry updated", payload={"entry_id": entry.id}) return entry # #endregion DictionaryManager.edit_entry @@ -251,10 +254,10 @@ class DictionaryManager: entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() if not entry: raise ValueError(f"Entry not found: {entry_id}") - logger.reason("Deleting dictionary entry", {"entry_id": entry_id}) + log.reason("Deleting dictionary entry", payload={"entry_id": entry_id}) db.delete(entry) db.commit() - logger.reflect("Entry deleted", {"entry_id": entry_id}) + log.reflect("Entry deleted", payload={"entry_id": entry_id}) # #endregion DictionaryManager.delete_entry # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear] @@ -265,10 +268,10 @@ class DictionaryManager: dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary not found: {dict_id}") - logger.reason("Clearing all entries", {"dict_id": dict_id}) + log.reason("Clearing all entries", payload={"dict_id": dict_id}) deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete() db.commit() - logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted}) + log.reflect("Entries cleared", payload={"dict_id": dict_id, "count": deleted}) return deleted # #endregion DictionaryManager.clear_entries @@ -316,7 +319,7 @@ class DictionaryManager: # Detect delimiter if not specified if not delimiter: delimiter = _detect_delimiter(content) - logger.reason("Detected delimiter", {"delimiter": repr(delimiter)}) + log.reason("Detected delimiter", payload={"delimiter": repr(delimiter)}) if delimiter not in (",", "\t"): raise ValueError(f"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\t'.") @@ -423,7 +426,7 @@ class DictionaryManager: if not preview_only: db.commit() - logger.reflect("Import complete", { + log.reflect("Import complete", payload={ "dict_id": dict_id, "total": result["total"], "created": result["created"], @@ -451,7 +454,7 @@ class DictionaryManager: .all() ) if not job_dict_links: - logger.reason("No dictionaries attached to job", {"job_id": job_id}) + log.reason("No dictionaries attached to job", payload={"job_id": job_id}) return [] dict_ids = [jd.dictionary_id for jd in job_dict_links] @@ -522,7 +525,7 @@ class DictionaryManager: dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)} matched.sort(key=lambda m: (dict_priority.get(m["dictionary_id"], 999), m["text_index"])) - logger.reflect("Batch filter match complete", { + log.reflect("Batch filter match complete", payload={ "job_id": job_id, "source_texts": len(source_texts), "matches": len(matched), @@ -625,7 +628,7 @@ class DictionaryManager: result["message"] = f"Entry created for '{source_term}' -> '{corrected_target_term}'" db.commit() - logger.reflect("Correction processed", result) + log.reflect("Correction processed", payload=result) return result # #endregion DictionaryManager.submit_correction diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index 6f58bfa2..e36fd3a5 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -16,9 +16,12 @@ from sqlalchemy.orm import Session from datetime import datetime, timezone, timedelta import uuid -from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger +from ...core.logger import belief_scope from ...models.translate import TranslationEvent, MetricSnapshot +log = MarkerLogger("EventLog") + # Terminal events per run — exactly one of these must exist for a completed run. TERMINAL_EVENT_TYPES = {"RUN_COMPLETED", "RUN_FAILED", "RUN_CANCELLED"} # All valid event types for validation. @@ -133,7 +136,7 @@ class TranslationEventLog: run_id=run_id, created_by=created_by, ) - logger.reason(f"Event logged: {event_type}", { + log.reason("Event logged", payload={ "event_id": event.id, "job_id": job_id, "run_id": run_id, @@ -195,7 +198,7 @@ class TranslationEventLog: ) -> Dict[str, Any]: with belief_scope("TranslationEventLog.prune_expired"): cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) - logger.reason("Pruning expired events", { + log.reason("Pruning expired events", payload={ "cutoff": cutoff.isoformat(), "retention_days": retention_days, }) @@ -207,7 +210,7 @@ class TranslationEventLog: total_expired = expired_query.count() if total_expired == 0: - logger.reflect("No expired events to prune", {}) + log.reflect("No expired events to prune", payload={}) return {"pruned": 0, "snapshot_id": None} # Create MetricSnapshot before pruning @@ -240,11 +243,11 @@ class TranslationEventLog: ).delete(synchronize_session=False) self.db.flush() pruned += len(ids) - logger.reason("Pruned batch", {"batch_size": len(ids), "total_pruned": pruned}) + log.reason("Pruned batch", payload={"batch_size": len(ids), "total_pruned": pruned}) self.db.commit() - logger.reflect("Pruning complete", { + log.reflect("Pruning complete", payload={ "pruned": pruned, "snapshot_id": snapshot_id, }) diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index 972268c8..40b30747 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Callable from sqlalchemy.orm import Session from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...models.translate import ( TranslationJob, @@ -39,6 +40,7 @@ from ...core.superset_client import SupersetClient from .dictionary import DictionaryManager from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE +log = MarkerLogger("TranslationExecutor") # #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry] MAX_RETRIES_PER_BATCH = 3 @@ -83,7 +85,7 @@ class TranslationExecutor: if not job: raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'") - logger.reason("Starting translation execution", { + log.reason("Starting translation execution", payload={ "run_id": run.id, "job_id": job.id, "batch_size": job.batch_size, @@ -103,7 +105,7 @@ class TranslationExecutor: # Preview-based: fetch rows from the accepted preview session source_rows = self._fetch_source_rows(job.id, run.id) if not source_rows: - logger.explore("No source rows to translate", {"run_id": run.id}) + log.explore("No source rows to translate", payload={"run_id": run.id}, error="Preview produced 0 source rows for translation") run.status = "COMPLETED" run.completed_at = datetime.now(timezone.utc) self.db.flush() @@ -119,7 +121,7 @@ class TranslationExecutor: for i in range(0, total_rows, batch_size) ] - logger.reason(f"Processing {len(batches)} batches", { + log.reason(f"Processing {len(batches)} batches", payload={ "run_id": run.id, "total_rows": total_rows, "batch_size": batch_size, @@ -163,7 +165,7 @@ class TranslationExecutor: run.completed_at = datetime.now(timezone.utc) self.db.flush() - logger.reflect("Translation execution complete", { + log.reflect("Translation execution complete", payload={ "run_id": run.id, "status": run.status, "total": total_rows, @@ -193,7 +195,7 @@ class TranslationExecutor: .first() ) if not session: - logger.explore("No accepted preview session found", {"job_id": job_id}) + log.explore("No accepted preview session found", error="Preview session has no accepted rows", payload={"job_id": job_id}) return [] # Fetch APPROVED or all records from the session @@ -216,7 +218,7 @@ class TranslationExecutor: "source_data": rec.source_data or {}, }) - logger.reason(f"Fetched {len(source_rows)} source rows from preview", { + log.reason(f"Fetched {len(source_rows)} source rows from preview", payload={ "run_id": run_id, "session_id": session.id, }) @@ -278,7 +280,8 @@ class TranslationExecutor: headers={"Content-Type": "application/json"}, ) except Exception as e: - logger.explore("Chart data API failed during full fetch", { + log.explore("Chart data API failed during full fetch", error="Chart data API error", + payload={ "offset": len(all_rows), "error": str(e), }) @@ -292,7 +295,7 @@ class TranslationExecutor: break # No more data all_rows.extend(rows) - logger.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", { + log.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", payload={ "offset": len(all_rows) - len(rows), }) @@ -319,7 +322,7 @@ class TranslationExecutor: "approved_translation": None, }) - logger.reason(f"Prepared {len(source_rows)} source rows for full translation", { + log.reason(f"Prepared {len(source_rows)} source rows for full translation", payload={ "job_id": job.id, }) return source_rows @@ -416,7 +419,7 @@ class TranslationExecutor: self.db.flush() batch_latency = int((time.monotonic() - batch_start) * 1000) - logger.reason(f"Batch {batch_index} complete", { + log.reason(f"Batch {batch_index} complete", payload={ "batch_id": batch_id, "latency_ms": batch_latency, **result, @@ -490,7 +493,8 @@ class TranslationExecutor: except Exception as e: last_error = str(e) retries += 1 - logger.explore(f"LLM call failed (attempt {attempt})", { + log.explore("LLM call failed", error="LLM call failed, retrying", + payload={ "batch_id": batch_id, "error": last_error, "attempt": attempt, @@ -498,7 +502,8 @@ class TranslationExecutor: if attempt < MAX_RETRIES_PER_BATCH: time.sleep(2 ** attempt) # Exponential backoff else: - logger.explore("LLM call exhausted retries", { + log.explore("LLM call exhausted retries", error="LLM retries exhausted", + payload={ "batch_id": batch_id, "last_error": last_error, }) @@ -527,7 +532,8 @@ class TranslationExecutor: translations = self._parse_llm_response(llm_response, len(batch_rows)) except ValueError as e: # Parse failure — mark all rows as SKIPPED - logger.explore("LLM response parse failed", { + log.explore("LLM response parse failed", error="Failed to parse LLM JSON response", + payload={ "batch_id": batch_id, "error": str(e), "response_preview": llm_response[:500] if llm_response else "", @@ -693,7 +699,7 @@ class TranslationExecutor: if provider_type in ("openai", "openai_compatible"): payload["response_format"] = {"type": "json_object"} - logger.reason( + log.reason( f"LLM request model={payload.get('model')} " f"provider_type={provider_type} " f"response_format={'yes' if 'response_format' in payload else 'no'} " @@ -701,11 +707,11 @@ class TranslationExecutor: ) response = http_requests.post(url, headers=headers, json=payload, timeout=180) if not response.ok: - logger.explore( - f"LLM API error status={response.status_code} " - f"model={payload.get('model')} " - f"body={response.text[:2000]}" - ) + log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={ + "status_code": response.status_code, + "model": payload.get('model'), + "body": response.text[:2000], + }) response.raise_for_status() data = response.json() @@ -717,7 +723,8 @@ class TranslationExecutor: if not content: # Log full response for diagnostics finish_reason = choices[0].get("finish_reason", "unknown") - logger.explore("LLM returned empty content", { + log.explore("LLM returned empty content", error="Empty response from LLM", + payload={ "finish_reason": finish_reason, "model": payload.get("model"), "prompt_len": len(prompt), diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index a4e8cc7d..ae1e54a0 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -26,6 +26,7 @@ from typing import Any, Dict, List, Optional, Callable, Tuple from sqlalchemy.orm import Session from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...models.translate import ( TranslationJob, @@ -41,6 +42,7 @@ from .superset_executor import SupersetSqlLabExecutor from .events import TranslationEventLog from ..translate.service import TranslateJobService +log = MarkerLogger("TranslationOrchestrator") # #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator] # @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. @@ -81,7 +83,7 @@ class TranslationOrchestrator: raise ValueError(f"Translation job '{job_id}' not found") self._job = job - logger.reason("Starting translation run", { + log.reason("Starting translation run", payload={ "job_id": job_id, "is_scheduled": is_scheduled, }) @@ -156,7 +158,7 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - logger.reflect("Run created", { + log.reflect("Run created", payload={ "run_id": run.id, "job_id": job_id, "status": run.status, @@ -189,7 +191,7 @@ class TranslationOrchestrator: f"Run must be in PENDING status." ) - logger.reason("Executing run", { + log.reason("Executing run", payload={ "run_id": run.id, "job_id": job.id, "skip_insert": skip_insert, @@ -212,9 +214,8 @@ class TranslationOrchestrator: try: run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation) except Exception as e: - logger.explore("Translation execution failed", { + log.explore("Translation execution failed", error=str(e), payload={ "run_id": run.id, - "error": str(e), }) run.status = "FAILED" run.error_message = f"Translation execution failed: {e}" @@ -291,7 +292,7 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - logger.reflect("Run execution complete", { + log.reflect("Run execution complete", payload={ "run_id": run.id, "status": run.status, "insert_status": run.insert_status, @@ -322,10 +323,10 @@ class TranslationOrchestrator: ) if not records: - logger.reason("No successful records to insert", {"run_id": run.id}) + log.reason("No successful records to insert", payload={"run_id": run.id}) return {"status": "skipped", "reason": "no_records", "query_id": None} - logger.reason(f"Generating SQL for {len(records)} records", { + log.reason(f"Generating SQL for {len(records)} records", payload={ "run_id": run.id, "dialect": job.database_dialect or job.target_dialect, }) @@ -371,11 +372,11 @@ class TranslationOrchestrator: upsert_strategy=job.upsert_strategy or "MERGE", ) except ValueError as e: - logger.explore("SQL generation failed", {"error": str(e)}) + log.explore("SQL generation failed", error=str(e)) return {"status": "failed", "error_message": str(e), "query_id": None} # Log insert phase start - logger.reason("Insert SQL generated", { + log.reason("Insert SQL generated", payload={ "run_id": run.id, "sql_preview": sql[:500], "sql_length": len(sql), @@ -404,7 +405,7 @@ class TranslationOrchestrator: except (ValueError, TypeError): # Could be a UUID — pass as-is, executor will resolve it target_db_id = job.target_database_id - logger.reason("target_database_id is not an integer, will resolve as UUID", { + log.reason("target_database_id is not an integer, will resolve as UUID", payload={ "target_database_id": job.target_database_id, }) executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id) @@ -414,9 +415,8 @@ class TranslationOrchestrator: poll_interval_seconds=2.0, ) except Exception as e: - logger.explore("Superset SQL submission failed", { + log.explore("Superset SQL submission failed", error=str(e), payload={ "run_id": run.id, - "error": str(e), }) result = {"status": "failed", "error_message": str(e), "query_id": None} @@ -488,7 +488,7 @@ class TranslationOrchestrator: "Run and accept a preview before executing a manual translation run." ) - logger.reason("Preconditions validated", { + log.reason("Preconditions validated", payload={ "job_id": job.id, "is_scheduled": is_scheduled, }) @@ -526,7 +526,7 @@ class TranslationOrchestrator: if not failed_batches: raise ValueError(f"No failed batches found for run '{run_id}'") - logger.reason("Retrying failed batches", { + log.reason("Retrying failed batches", payload={ "run_id": run_id, "batch_count": len(failed_batches), }) @@ -602,7 +602,7 @@ class TranslationOrchestrator: ) self.db.commit() - logger.reflect("Retry complete", { + log.reflect("Retry complete", payload={ "run_id": run_id, "status": run.status, }) @@ -628,7 +628,7 @@ class TranslationOrchestrator: raise ValueError(f"Job '{run.job_id}' not found") self._job = job - logger.reason("Retrying insert phase", { + log.reason("Retrying insert phase", payload={ "run_id": run_id, }) @@ -652,7 +652,7 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - logger.reflect("Insert retry complete", { + log.reflect("Insert retry complete", payload={ "run_id": run_id, "insert_status": run.insert_status, }) @@ -692,7 +692,7 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - logger.reflect("Run cancelled", { + log.reflect("Run cancelled", payload={ "run_id": run_id, }) return run diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index b95b2953..ece466b7 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -25,7 +25,8 @@ import json import hashlib import re -from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger +from ...core.logger import belief_scope from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient from ...models.translate import ( @@ -38,6 +39,8 @@ from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt from .dictionary import DictionaryManager +log = MarkerLogger("TranslationPreview") + # #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt] DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( "Translate the following database content from {source_language} to {target_language}.\n\n" @@ -137,7 +140,7 @@ class TranslationPreview: env_id: Optional[str] = None, ) -> Dict[str, Any]: with belief_scope("TranslationPreview.preview_rows"): - logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size}) + log.reason("Starting preview for job", payload={"job_id": job_id, "sample_size": sample_size}) # 1. Load job job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() @@ -153,7 +156,7 @@ class TranslationPreview: dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id) # 3. Fetch sample rows from Superset - logger.reason("Fetching sample rows from Superset", { + log.reason("Fetching sample rows from Superset", payload={ "datasource_id": job.source_datasource_id, "sample_size": sample_size, "translation_column": job.translation_column, @@ -167,12 +170,12 @@ class TranslationPreview: raise ValueError("No rows returned from datasource for preview") actual_row_count = len(source_rows) - logger.reason(f"Fetched {actual_row_count} sample row(s)") + log.reason(f"Fetched {actual_row_count} sample row(s)") # Debug: log first row keys and translation column value if source_rows: first_row = source_rows[0] - logger.reason( + log.reason( f"First source row keys={list(first_row.keys())} " f"translation_col={job.translation_column} " f"val='{first_row.get(job.translation_column, '')}'" @@ -246,7 +249,7 @@ class TranslationPreview: total_est_cost = TokenEstimator.estimate_cost(total_est_tokens) # 8. Call LLM - logger.reason("Calling LLM for preview translation", { + log.reason("Calling LLM for preview translation", payload={ "provider_id": job.provider_id, "row_count": actual_row_count, "estimated_tokens": sample_total_tokens, @@ -330,7 +333,7 @@ class TranslationPreview: "dict_snapshot_hash": dict_snapshot_hash, } - logger.reflect("Preview completed", { + log.reflect("Preview completed", payload={ "session_id": session.id, "row_count": actual_row_count, "sample_cost": sample_cost, @@ -393,7 +396,7 @@ class TranslationPreview: self.db.commit() self.db.refresh(record) - logger.reason(f"Preview row {action}d", { + log.reason(f"Preview row {action}d", payload={ "row_id": row_id, "session_id": session.id, "status": record.status, @@ -431,7 +434,7 @@ class TranslationPreview: self.db.commit() self.db.refresh(session) - logger.reason("Preview session accepted", { + log.reason("Preview session accepted", payload={ "session_id": session.id, "job_id": job_id, }) @@ -521,13 +524,15 @@ class TranslationPreview: None, ) if not env_config: - logger.explore("Could not find environment for datasource", { + log.explore("Could not find environment for datasource", error="Environment not found for datasource", + payload={ "env_id": target_env_id, }) # Fallback: try first environment if environments: env_config = environments[0] - logger.explore("Falling back to first available environment", { + log.explore("Falling back to first available environment", error="Environment fallback used", + payload={ "env_id": env_config.id, }) else: @@ -572,17 +577,17 @@ class TranslationPreview: headers={"Content-Type": "application/json"}, ) except Exception as e: - logger.explore("Chart data API failed", {"error": str(e)}) + log.explore("Chart data API failed", error=str(e)) raise ValueError(f"Failed to fetch sample data from Superset: {e}") # Parse response rows = self._extract_data_rows(response) - logger.reason(f"Extracted {len(rows)} data row(s)") + log.reason(f"Extracted {len(rows)} data row(s)") # Debug: log first row keys and translation column value if rows: first_row = rows[0] - logger.reason( + log.reason( f"Row keys={list(first_row.keys())} " f"target_col={job.translation_column} " f"val='{first_row.get(job.translation_column, '')}'" @@ -658,7 +663,7 @@ class TranslationPreview: else: raise ValueError(f"Unsupported provider type '{provider_type}' for preview") - logger.reason("LLM call completed", { + log.reason("LLM call completed", payload={ "provider_id": job.provider_id, "model": model, "response_length": len(response_text), @@ -701,7 +706,7 @@ class TranslationPreview: if provider_type in ("openai", "openai_compatible"): payload["response_format"] = {"type": "json_object"} - logger.reason( + log.reason( f"LLM request model={payload.get('model')} " f"provider_type={provider_type} " f"response_format={'yes' if 'response_format' in payload else 'no'} " @@ -709,11 +714,11 @@ class TranslationPreview: ) response = http_requests.post(url, headers=headers, json=payload, timeout=120) if not response.ok: - logger.explore( - f"LLM API error status={response.status_code} " - f"model={payload.get('model')} " - f"body={response.text[:2000]}" - ) + log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={ + "status_code": response.status_code, + "model": payload.get('model'), + "body": response.text[:2000], + }) response.raise_for_status() data = response.json() @@ -736,7 +741,7 @@ class TranslationPreview: @staticmethod def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: with belief_scope("TranslationPreview._parse_llm_response"): - logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}") + log.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}") try: data = json.loads(response_text) @@ -754,7 +759,7 @@ class TranslationPreview: rows = data.get("rows", []) if not isinstance(rows, list): - logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}") + log.explore("LLM response has no 'rows' array", error="LLM response missing 'rows' key") raise ValueError("LLM response missing 'rows' array") translations: Dict[str, str] = {} @@ -765,10 +770,11 @@ class TranslationPreview: translations[row_id] = translation if len(translations) < expected_count: - logger.explore( - f"LLM returned fewer translations expected={expected_count} " - f"got={len(translations)} response_preview={response_text[:600]}" - ) + log.explore("LLM returned fewer translations than expected", error=f"Expected {expected_count} translations, got {len(translations)}", payload={ + "expected_count": expected_count, + "actual_count": len(translations), + "response_preview": response_text[:600], + }) return translations # #endregion _parse_llm_response diff --git a/backend/src/plugins/translate/scheduler.py b/backend/src/plugins/translate/scheduler.py index 9f70061e..87268916 100644 --- a/backend/src/plugins/translate/scheduler.py +++ b/backend/src/plugins/translate/scheduler.py @@ -20,11 +20,14 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session -from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger +from ...core.logger import belief_scope from ...core.config_manager import ConfigManager from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun from .events import TranslationEventLog +log = MarkerLogger("TranslationScheduler") + # #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud] # @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers. @@ -82,7 +85,7 @@ class TranslationScheduler: created_by=self.current_user, ) - logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id}) + log.reflect("Schedule created", payload={"schedule_id": schedule.id, "job_id": job_id}) return schedule # #endregion create_schedule @@ -127,7 +130,7 @@ class TranslationScheduler: created_by=self.current_user, ) - logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id}) + log.reflect("Schedule updated", payload={"schedule_id": schedule.id, "job_id": job_id}) return schedule # #endregion update_schedule @@ -155,7 +158,7 @@ class TranslationScheduler: created_by=self.current_user, ) - logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id}) + log.reflect("Schedule deleted", payload={"schedule_id": schedule_id, "job_id": job_id}) # #endregion delete_schedule # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate] @@ -176,7 +179,7 @@ class TranslationScheduler: self.db.commit() self.db.refresh(schedule) - logger.reflect("Schedule active state set", { + log.reflect("Schedule active state set", payload={ "schedule_id": schedule.id, "job_id": job_id, "is_active": is_active, @@ -218,7 +221,7 @@ class TranslationScheduler: tz = ZoneInfo(timezone_str) trigger = CronTrigger.from_crontab(cron_expression, timezone=tz) except (ValueError, KeyError) as e: - logger.warning(f"[get_next_executions] Invalid cron: {e}") + log.explore("Invalid cron", error=str(e)) return [] now = datetime.now(tz) @@ -263,10 +266,10 @@ def execute_scheduled_translation( TranslationSchedule.job_id == job_id, ).first() if not schedule or not schedule.is_active: - logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}") + log.reason("Schedule inactive/missing", payload={"schedule_id": schedule_id}) return - logger.reason("Scheduled translation triggered", { + log.reason("Scheduled translation triggered", payload={ "schedule_id": schedule_id, "job_id": job_id, }) @@ -281,7 +284,8 @@ def execute_scheduled_translation( .first() ) if active_run: - logger.explore("Skipping scheduled run — concurrent run in progress", { + log.explore("Skipping scheduled run — concurrent run in progress", error="Concurrent run detected", + payload={ "job_id": job_id, "existing_run_id": active_run.id, }) @@ -308,7 +312,7 @@ def execute_scheduled_translation( age = datetime.now(timezone.utc) - most_recent.created_at if age > timedelta(days=90): baseline_expired = True - logger.reason("Baseline expired — full translation", { + log.reason("Baseline expired — full translation", payload={ "job_id": job_id, "last_run": most_recent.created_at.isoformat(), "age_days": age.days, @@ -339,10 +343,7 @@ def execute_scheduled_translation( orch.execute_run(run) run.insert_status = run.insert_status or "succeeded" except Exception as exec_err: - logger.explore("Scheduled translation execution failed", { - "run_id": run.id, - "error": str(exec_err), - }) + log.explore("Scheduled translation execution failed", error=str(exec_err)) # Leave schedule enabled — schedule continues on failure run.status = "FAILED" run.error_message = str(exec_err) @@ -352,13 +353,13 @@ def execute_scheduled_translation( schedule.last_run_at = datetime.now(timezone.utc) db.commit() - logger.reflect("Scheduled translation complete", { + log.reflect("Scheduled translation complete", payload={ "schedule_id": schedule_id, "run_id": run.id, "status": run.status, }) except Exception as e: - logger.error(f"[scheduled_translation] Unexpected error: {e}") + log.explore("Unexpected error", error=str(e)) finally: db.close() # #endregion execute_scheduled_translation diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index fb0441bb..a01d4a30 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone import uuid from ...core.logger import logger +from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient from ...models.translate import TranslationJob, TranslationJobDictionary @@ -29,6 +30,8 @@ from ...schemas.translate import ( DatasourceColumnResponse, ) +log = MarkerLogger("TranslateJobService") + # Supported database dialects for translation SUPPORTED_DIALECTS = { "postgresql", "mysql", "clickhouse", "sqlite", "mssql", @@ -44,7 +47,7 @@ SUPPORTED_DIALECTS = { # @SIDE_EFFECT None — pure data extraction. def get_dialect_from_database(database_record: Dict[str, Any]) -> str: """Extract and validate dialect from Superset database record.""" - logger.reason("Extracting dialect from database record", { + log.reason("Extracting dialect from database record", payload={ "has_backend": bool(database_record.get("backend") or database_record.get("engine")), }) backend = ( @@ -54,7 +57,7 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str: ).lower().strip() if not backend: - logger.explore("Could not determine database dialect", { + log.explore("Could not determine database dialect", error="No backend/engine field in database record", payload={ "record_keys": list(database_record.keys()), }) raise ValueError("Could not determine database dialect from connection") @@ -82,12 +85,12 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str: normalized = dialect_map.get(backend, backend) if normalized not in SUPPORTED_DIALECTS: - logger.explore("Unsupported dialect", {"backend": backend, "normalized": normalized}) + log.explore("Unsupported dialect", payload={"backend": backend, "normalized": normalized}, error=f"Unsupported database dialect: {backend}") raise ValueError( f"Unsupported database dialect: '{backend}'. " f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}" ) - logger.reflect("Dialect validated", {"dialect": normalized}) + log.reflect("Dialect validated", payload={"dialect": normalized}) return normalized # #endregion get_dialect_from_database @@ -103,7 +106,7 @@ def fetch_datasource_metadata( config_manager: ConfigManager, ) -> Tuple[List[Dict[str, Any]], str]: """Fetch column metadata and database dialect for a datasource from Superset.""" - logger.reason("Fetching datasource metadata", { + log.reason("Fetching datasource metadata", payload={ "dataset_id": dataset_id, "env_id": env_id, }) @@ -111,7 +114,7 @@ def fetch_datasource_metadata( environments = config_manager.get_environments() env_config = next((e for e in environments if e.id == env_id), None) if not env_config: - logger.explore("Environment not found", {"env_id": env_id}) + log.explore("Environment not found", payload={"env_id": env_id}, error=f"Environment {env_id} not configured") raise ValueError(f"Superset environment '{env_id}' not found in configuration") # Create Superset client and fetch dataset detail @@ -148,10 +151,10 @@ def fetch_datasource_metadata( else: raise ValueError("No database information available for this datasource") except Exception as e: - logger.explore("Could not fetch database dialect", {"error": str(e)}) + log.explore("Could not fetch database dialect", error=str(e)) raise ValueError(f"Could not determine database dialect: {e}") - logger.reflect("Metadata fetched", {"columns": len(columns), "dialect": dialect}) + log.reflect("Metadata fetched", payload={"columns": len(columns), "dialect": dialect}) return columns, dialect # #endregion fetch_datasource_metadata @@ -213,11 +216,11 @@ class TranslateJobService: # @POST Returns the created TranslationJob with database_dialect cached. # @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect. def create_job(self, payload: TranslateJobCreate) -> TranslationJob: - logger.reason("Creating translation job", {"name": payload.name}) + log.reason("Creating translation job", payload={"name": payload.name}) # Validate: must have a translation column if datasource is configured if payload.source_datasource_id and not payload.translation_column: - logger.explore("Missing translation column", { + log.explore("Missing translation column", error="translation_column required when source_datasource_id is set", payload={ "source_datasource_id": payload.source_datasource_id, }) raise ValueError("A translation column is required when a datasource is selected") @@ -244,7 +247,7 @@ class TranslateJobService: ) dialect = detected_dialect except Exception as e: - logger.explore("Dialect detection failed", {"error": str(e)}) + log.explore("Dialect detection failed", error=str(e)) dialect = payload.source_dialect # Build job instance @@ -287,7 +290,7 @@ class TranslateJobService: self.db.commit() self.db.refresh(job) - logger.reflect("Job created", {"job_id": job.id}) + log.reflect("Job created", payload={"job_id": job.id}) return job # #endregion create_job @@ -297,7 +300,7 @@ class TranslateJobService: # @POST Returns the updated TranslationJob with re-detected dialect if datasource changed. # @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed. def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob: - logger.reason("Updating translation job", {"job_id": job_id}) + log.reason("Updating translation job", payload={"job_id": job_id}) job = self.get_job(job_id) update_data = payload.model_dump(exclude_unset=True) @@ -318,7 +321,7 @@ class TranslateJobService: ) job.database_dialect = detected_dialect except Exception as e: - logger.explore("Dialect re-detection failed", {"error": str(e)}) + log.explore("Dialect re-detection failed", error=str(e)) job.updated_at = datetime.now(timezone.utc) @@ -337,7 +340,7 @@ class TranslateJobService: self.db.commit() self.db.refresh(job) - logger.reflect("Job updated", {"job_id": job_id}) + log.reflect("Job updated", payload={"job_id": job_id}) return job # #endregion update_job @@ -347,7 +350,7 @@ class TranslateJobService: # @POST Job and all related TranslationJobDictionary records are deleted. # @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows. def delete_job(self, job_id: str) -> None: - logger.reason("Deleting translation job", {"job_id": job_id}) + log.reason("Deleting translation job", payload={"job_id": job_id}) job = self.get_job(job_id) # Delete dictionary associations @@ -357,7 +360,7 @@ class TranslateJobService: self.db.delete(job) self.db.commit() - logger.reflect("Job deleted", {"job_id": job_id}) + log.reflect("Job deleted", payload={"job_id": job_id}) # #endregion delete_job # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate] @@ -366,7 +369,7 @@ class TranslateJobService: # @POST Returns the duplicated TranslationJob with status DRAFT. # @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows. def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob: - logger.reason("Duplicating translation job", {"job_id": job_id, "new_name": new_name}) + log.reason("Duplicating translation job", payload={"job_id": job_id, "new_name": new_name}) source = self.get_job(job_id) # Copy all fields except id, created_at, updated_at, status @@ -411,7 +414,7 @@ class TranslateJobService: self.db.commit() self.db.refresh(new_job) - logger.reflect("Job duplicated", {"new_job_id": new_job.id, "source_job_id": job_id}) + log.reflect("Job duplicated", payload={"new_job_id": new_job.id, "source_job_id": job_id}) return new_job # #endregion duplicate_job @@ -512,13 +515,13 @@ def get_datasource_columns( config_manager: ConfigManager, ) -> DatasourceColumnsResponse: """Fetch and return column metadata for a given Superset datasource.""" - logger.reason("Fetching datasource columns", {"datasource_id": datasource_id, "env_id": env_id}) + log.reason("Fetching datasource columns", payload={"datasource_id": datasource_id, "env_id": env_id}) # Find environment config environments = config_manager.get_environments() env_config = next((e for e in environments if e.id == env_id), None) if not env_config: - logger.explore("Environment not found", {"env_id": env_id}) + log.explore("Environment not found", payload={"env_id": env_id}, error=f"Environment {env_id} not configured") raise ValueError(f"Superset environment '{env_id}' not found") # Create Superset client @@ -540,7 +543,7 @@ def get_datasource_columns( db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record dialect = get_dialect_from_database(db_result) else: - logger.explore("Could not determine dialect", {"datasource_id": datasource_id}) + log.explore("Could not determine dialect", payload={"datasource_id": datasource_id}, error=f"Could not determine dialect for datasource {datasource_id}") raise ValueError("Could not determine database dialect for this datasource") # Extract columns @@ -565,7 +568,7 @@ def get_datasource_columns( database_dialect=dialect, columns=columns, ) - logger.reflect("Datasource columns fetched", { + log.reflect("Datasource columns fetched", payload={ "datasource_id": datasource_id, "columns_count": len(columns), "dialect": dialect, diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py index 3e8e46b9..15ca71af 100644 --- a/backend/src/plugins/translate/sql_generator.py +++ b/backend/src/plugins/translate/sql_generator.py @@ -8,7 +8,11 @@ from typing import Any, Dict, List, Optional, Tuple from datetime import datetime, timezone -from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger + +from ...core.logger import belief_scope + +log = MarkerLogger("SqlGenerator") # PostgreSQL/Greenplum dialects that support ON CONFLICT POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"} @@ -240,7 +244,7 @@ class SQLGenerator: Tuple of (sql_string, row_count). """ with belief_scope("SQLGenerator.generate"): - logger.reason("Generating SQL", { + log.reason("Generating SQL", payload={ "dialect": dialect, "schema": target_schema, "table": target_table, @@ -306,7 +310,7 @@ class SQLGenerator: dialect=dialect, ) if use_upsert: - logger.reason("ClickHouse UPSERT not supported; using plain INSERT", { + log.reason("ClickHouse UPSERT not supported; using plain INSERT", payload={ "note": "ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.", }) else: @@ -319,7 +323,7 @@ class SQLGenerator: dialect=dialect, ) - logger.reflect("SQL generated", { + log.reflect("SQL generated", payload={ "dialect": dialect, "row_count": len(rows), "sql_length": len(sql), diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index b1fbc9df..cafc2fd4 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -18,9 +18,11 @@ import time import uuid from ...core.logger import logger, belief_scope +from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient +log = MarkerLogger("SupersetExecutor") # #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab] # @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking. @@ -74,7 +76,7 @@ class SupersetSqlLabExecutor: for db in databases: if db.get("database_name", "").lower() == database_name.lower(): self._database_id = db["id"] - logger.reason("Resolved database ID by name", { + log.reason("Resolved database ID by name", payload={ "database_name": database_name, "database_id": self._database_id, }) @@ -85,7 +87,7 @@ class SupersetSqlLabExecutor: # Default: use first database self._database_id = databases[0]["id"] - logger.reason("Using default database", { + log.reason("Using default database", payload={ "database_name": databases[0].get("database_name"), "database_id": self._database_id, }) @@ -111,7 +113,7 @@ class SupersetSqlLabExecutor: if db_uuid.lower() == uuid_str.lower(): db_id = db["id"] self._database_id = db_id - logger.reason("Resolved database ID by UUID", { + log.reason("Resolved database ID by UUID", payload={ "uuid": uuid_str, "database_id": db_id, "database_name": db.get("database_name"), @@ -145,7 +147,7 @@ class SupersetSqlLabExecutor: except (ValueError, TypeError): db_id = self.resolve_database_uuid(db_id) - logger.reason("Submitting SQL to Superset SQL Lab", { + log.reason("Submitting SQL to Superset SQL Lab", payload={ "database_id": db_id, "sql_length": len(sql), }) @@ -165,7 +167,7 @@ class SupersetSqlLabExecutor: headers={"Content-Type": "application/json"}, ) except Exception as e: - logger.explore("SQL Lab execute failed", {"error": str(e)}) + log.explore("SQL Lab execute failed", error=str(e)) raise ValueError(f"Superset SQL Lab execute failed: {e}") # Parse response — try multiple formats @@ -177,7 +179,7 @@ class SupersetSqlLabExecutor: or (result.get("result") or {}).get("id")) status = result.get("status", "unknown") - logger.reason("SQL Lab execute response", { + log.reason("SQL Lab execute response", payload={ "query_id": query_id, "status": status, "has_query_id": query_id is not None, @@ -208,7 +210,7 @@ class SupersetSqlLabExecutor: with belief_scope("SupersetSqlLabExecutor.poll_execution_status"): client = self._get_client() - logger.reason("Polling SQL execution status", { + log.reason("Polling SQL execution status", payload={ "query_id": query_id, "max_polls": max_polls, "interval": poll_interval_seconds, @@ -226,7 +228,7 @@ class SupersetSqlLabExecutor: # Terminal states if state in ("success", "finished", "completed"): - logger.reason("SQL execution completed", { + log.reason("SQL execution completed", payload={ "query_id": query_id, "attempt": attempt + 1, "rows_affected": result.get("rows"), @@ -243,7 +245,8 @@ class SupersetSqlLabExecutor: if state in ("failed", "error", "stopped"): error_msg = result.get("error_message", "Unknown error") - logger.explore("SQL execution failed", { + log.explore("SQL execution failed", error="SQL query execution returned failed/error state", + payload={ "query_id": query_id, "state": state, "error": error_msg, @@ -264,7 +267,8 @@ class SupersetSqlLabExecutor: time.sleep(poll_interval_seconds) except Exception as e: - logger.explore("Polling error, retrying", { + log.explore("Polling error, retrying", error="Polling encountered error, will retry", + payload={ "query_id": query_id, "attempt": attempt + 1, "error": str(e), @@ -273,7 +277,7 @@ class SupersetSqlLabExecutor: continue # Timeout - logger.explore("SQL execution polling timed out", { + log.explore("SQL execution polling timed out", error="Polling timed out", payload={ "query_id": query_id, "max_polls": max_polls, }) @@ -305,7 +309,7 @@ class SupersetSqlLabExecutor: query_id = exec_result.get("query_id") if not query_id: - logger.explore("No query_id from SQL Lab execute", { + log.explore("No query_id from SQL Lab execute", error="No query_id returned", payload={ "raw_response": exec_result.get("raw_response"), }) return { @@ -341,7 +345,8 @@ class SupersetSqlLabExecutor: "results": result.get("results"), } except Exception as e: - logger.explore("Failed to fetch query results", { + log.explore("Failed to fetch query results", error="Failed to fetch query results from Superset", + payload={ "query_id": query_id, "error": str(e), }) diff --git a/backend/src/scripts/migrate_sqlite_to_postgres.py b/backend/src/scripts/migrate_sqlite_to_postgres.py index 0a74d11a..c9b190f5 100644 --- a/backend/src/scripts/migrate_sqlite_to_postgres.py +++ b/backend/src/scripts/migrate_sqlite_to_postgres.py @@ -23,7 +23,10 @@ from typing import Any, Dict, Iterable, Optional from sqlalchemy import create_engine, text from sqlalchemy.exc import SQLAlchemyError -from src.core.logger import belief_scope, logger +from src.core.logger import belief_scope +from src.core.cot_logger import MarkerLogger +log = MarkerLogger("MigrateSqliteToPostgres") + # [/SECTION] @@ -169,9 +172,7 @@ def _ensure_target_schema(engine) -> None: def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int: with belief_scope("_migrate_config"): if legacy_config_path is None: - logger.info( - "[_migrate_config][Action] No legacy config.json found, skipping" - ) + log.reason("No legacy config.json found, skipping migration") return 0 payload = json.loads(legacy_config_path.read_text(encoding="utf-8")) @@ -332,9 +333,7 @@ def run_migration( sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path] ) -> Dict[str, int]: with belief_scope("run_migration"): - logger.info( - "[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url - ) + log.reason("Starting migration from SQLite to PostgreSQL", payload={"sqlite_path": str(sqlite_path), "target_url": target_url}) if not sqlite_path.exists(): raise FileNotFoundError(f"SQLite source not found: {sqlite_path}") diff --git a/backend/src/scripts/seed_superset_load_test.py b/backend/src/scripts/seed_superset_load_test.py index 66e453c0..16fa3dc7 100644 --- a/backend/src/scripts/seed_superset_load_test.py +++ b/backend/src/scripts/seed_superset_load_test.py @@ -21,8 +21,11 @@ sys.path.append(str(Path(__file__).parent.parent.parent)) from src.core.config_manager import ConfigManager from src.core.config_models import Environment -from src.core.logger import belief_scope, logger +from src.core.logger import belief_scope +from src.core.cot_logger import MarkerLogger from src.core.superset_client import SupersetClient +log = MarkerLogger("SeedSupersetLoadTest") + # [/SECTION] @@ -161,9 +164,7 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]: env = Environment(**row) configured[env.id] = env except Exception as exc: - logger.warning( - f"[REFLECT] Failed loading environments from {config_path}: {exc}" - ) + log.explore("Failed loading environments from config", payload={"config_path": str(config_path)}, error=str(exc)) for env_id in env_ids: env = configured.get(env_id) @@ -275,9 +276,7 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: templates_by_env[env_id] = _build_chart_template_pool( client, args.template_pool_size, rng ) - logger.info( - f"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates" - ) + log.reason("Loaded chart templates for environment", payload={"env_id": env_id, "template_count": len(templates_by_env[env_id])}) errors = 0 env_ids = list(env_map.keys()) @@ -289,9 +288,7 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: dashboard_title = _generate_unique_name("lt_dash", used_dashboard_names, rng) if args.dry_run: - logger.info( - f"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}" - ) + log.reflect("Dry-run dashboard create", payload={"env_id": env_id, "title": dashboard_title}) continue try: @@ -305,7 +302,7 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: created_dashboards[env_id].append(dashboard_id) except Exception as exc: errors += 1 - logger.error(f"[EXPLORE] Failed creating dashboard in {env_id}: {exc}") + log.explore("Failed creating dashboard", payload={"env_id": env_id}, error=str(exc)) if errors >= args.max_errors: raise RuntimeError( f"Stopping due to max errors reached ({errors})" @@ -355,10 +352,10 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: created_charts[env_id].append(chart_id) if (index + 1) % 500 == 0: - logger.info(f"[REASON] Created {index + 1}/{args.charts} charts") + log.reason(f"Created {index + 1}/{args.charts} charts") except Exception as exc: errors += 1 - logger.error(f"[EXPLORE] Failed creating chart in {env_id}: {exc}") + log.explore("Failed creating chart", payload={"env_id": env_id}, error=str(exc)) if errors >= args.max_errors: raise RuntimeError( f"Stopping due to max errors reached ({errors})" @@ -385,9 +382,7 @@ def main() -> None: with belief_scope("seed_superset_load_test.main"): args = _parse_args() result = seed_superset_load_data(args) - logger.info( - f"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}" - ) + log.reflect("Seed load test complete", payload={"summary": result}) # [/DEF:main:Function] diff --git a/backend/src/services/dataset_review/clarification_engine.py b/backend/src/services/dataset_review/clarification_engine.py index 8f6167a8..44a62cec 100644 --- a/backend/src/services/dataset_review/clarification_engine.py +++ b/backend/src/services/dataset_review/clarification_engine.py @@ -23,7 +23,8 @@ from dataclasses import dataclass, field from datetime import datetime from typing import List, Optional -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.models.auth import User from src.models.dataset_review import ( AnswerKind, @@ -52,6 +53,8 @@ from src.services.dataset_review.clarification_pkg._helpers import ( derive_recommended_action, ) +log = MarkerLogger("ClarificationEngine") + # [DEF:ClarificationQuestionPayload:Class] # @COMPLEXITY: 2 @@ -130,7 +133,7 @@ class ClarificationEngine: with belief_scope("ClarificationEngine.build_question_payload"): clarification_session = self._get_latest_clarification_session(session) if clarification_session is None: - logger.reason("No clarification session found", extra={"session_id": session.session_id}) + log.reason("No clarification session found", payload={"session_id": session.session_id}) return None active_questions = [ @@ -146,7 +149,7 @@ class ClarificationEngine: if session.current_phase == SessionPhase.CLARIFICATION: session.current_phase = SessionPhase.REVIEW self.repository.db.commit() - logger.reflect("No unresolved clarification question remains", extra={"session_id": session.session_id}) + log.reflect("No unresolved clarification question remains", payload={"session_id": session.session_id}) return None selected_question = active_questions[0] @@ -156,7 +159,7 @@ class ClarificationEngine: session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION session.current_phase = SessionPhase.CLARIFICATION - logger.reason("Selected active clarification question", extra={"session_id": session.session_id, "question_id": selected_question.question_id, "priority": selected_question.priority}) + log.reason("Selected active clarification question", payload={"session_id": session.session_id, "question_id": selected_question.question_id, "priority": selected_question.priority}) self.repository.db.commit() payload = ClarificationQuestionPayload( @@ -173,7 +176,7 @@ class ClarificationEngine: for o in sorted(selected_question.options, key=lambda item: (item.display_order, item.label, item.option_id)) ], ) - logger.reflect("Clarification payload built", extra={"session_id": session.session_id, "question_id": payload.question_id, "option_count": len(payload.options)}) + log.reflect("Clarification payload built", payload={"session_id": session.session_id, "question_id": payload.question_id, "option_count": len(payload.options)}) return payload # [/DEF:build_question_payload:Function] @@ -189,25 +192,25 @@ class ClarificationEngine: session = command.session clarification_session = self._get_latest_clarification_session(session) if clarification_session is None: - logger.explore("Cannot record clarification answer because no clarification session exists", extra={"session_id": session.session_id}) + log.explore("Cannot record clarification answer because no clarification session exists", payload={"session_id": session.session_id}, error="No clarification session found") raise ValueError("Clarification session not found") question = self._find_question(clarification_session, command.question_id) if question is None: - logger.explore("Cannot record clarification answer for foreign or missing question", extra={"session_id": session.session_id, "question_id": command.question_id}) + log.explore("Cannot record clarification answer for foreign or missing question", payload={"session_id": session.session_id, "question_id": command.question_id}, error="Question not found in clarification session") raise ValueError("Clarification question not found") if question.answer is not None: - logger.explore("Rejected duplicate clarification answer submission", extra={"session_id": session.session_id, "question_id": command.question_id}) + log.explore("Rejected duplicate clarification answer submission", payload={"session_id": session.session_id, "question_id": command.question_id}, error="Question already answered") raise ValueError("Clarification question already answered") if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id: - logger.explore("Rejected answer for non-active clarification question", extra={"session_id": session.session_id, "question_id": question.question_id, "current_question_id": clarification_session.current_question_id}) + log.explore("Rejected answer for non-active clarification question", payload={"session_id": session.session_id, "question_id": question.question_id, "current_question_id": clarification_session.current_question_id}, error="Only active question can be answered") raise ValueError("Only the active clarification question can be answered") normalized_answer_value = normalize_answer_value(command.answer_kind, command.answer_value, question) - logger.reason("Persisting clarification answer before state advancement", extra={"session_id": session.session_id, "question_id": question.question_id, "answer_kind": command.answer_kind.value}) + log.reason("Persisting clarification answer before state advancement", payload={"session_id": session.session_id, "question_id": question.question_id, "answer_kind": command.answer_kind.value}) persisted_answer = ClarificationAnswer( question_id=question.question_id, answer_kind=command.answer_kind, @@ -254,7 +257,7 @@ class ClarificationEngine: self.repository.db.commit() self.repository.db.refresh(session) - logger.reflect("Clarification answer recorded and session advanced", extra={"session_id": session.session_id, "question_id": question.question_id, "next_question_id": clarification_session.current_question_id, "readiness_state": session.readiness_state.value, "remaining_count": clarification_session.remaining_count}) + log.reflect("Clarification answer recorded and session advanced", payload={"session_id": session.session_id, "question_id": question.question_id, "next_question_id": clarification_session.current_question_id, "readiness_state": session.readiness_state.value, "remaining_count": clarification_session.remaining_count}) return ClarificationStateResult( clarification_session=clarification_session, diff --git a/backend/src/services/dataset_review/event_logger.py b/backend/src/services/dataset_review/event_logger.py index de4face4..3af54c83 100644 --- a/backend/src/services/dataset_review/event_logger.py +++ b/backend/src/services/dataset_review/event_logger.py @@ -18,10 +18,13 @@ from typing import Any, Dict, Optional from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.models.dataset_review import DatasetReviewSession, SessionEvent # [/DEF:SessionEventLoggerImports:Block] +log = MarkerLogger("SessionEventLogger") + # [DEF:SessionEventPayload:Class] # @COMPLEXITY: 2 @@ -71,31 +74,22 @@ class SessionEventLogger: event_summary = str(payload.event_summary or "").strip() if not session_id: - logger.explore("Session event logging rejected because session_id is empty") + log.explore("Session event logging rejected because session_id is empty", error="session_id is empty") raise ValueError("session_id must be non-empty") if not actor_user_id: - logger.explore( - "Session event logging rejected because actor_user_id is empty", - extra={"session_id": session_id}, - ) + log.explore("Session event logging rejected because actor_user_id is empty", error="actor_user_id is empty", payload={"session_id": session_id}) raise ValueError("actor_user_id must be non-empty") if not event_type: - logger.explore( - "Session event logging rejected because event_type is empty", - extra={"session_id": session_id, "actor_user_id": actor_user_id}, - ) + log.explore("Session event logging rejected because event_type is empty", error="event_type is empty", payload={"session_id": session_id, "actor_user_id": actor_user_id}) raise ValueError("event_type must be non-empty") if not event_summary: - logger.explore( - "Session event logging rejected because event_summary is empty", - extra={"session_id": session_id, "event_type": event_type}, - ) + log.explore("Session event logging rejected because event_summary is empty", error="event_summary is empty", payload={"session_id": session_id, "event_type": event_type}) raise ValueError("event_summary must be non-empty") normalized_details = dict(payload.event_details or {}) - logger.reason( + log.reason( "Persisting explicit dataset-review session audit event", - extra={ + payload={ "session_id": session_id, "actor_user_id": actor_user_id, "event_type": event_type, @@ -117,9 +111,9 @@ class SessionEventLogger: self.db.commit() self.db.refresh(event) - logger.reflect( + log.reflect( "Dataset-review session audit event persisted", - extra={ + payload={ "session_id": session_id, "session_event_id": event.session_event_id, "event_type": event.event_type, diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index d63d7da6..19cdd0eb 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -25,7 +25,8 @@ from datetime import datetime from typing import Any, Dict, List, Optional, cast from src.core.config_manager import ConfigManager -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.core.task_manager import TaskManager from src.core.utils.superset_compilation_adapter import ( PreviewCompilationPayload, @@ -89,7 +90,7 @@ from src.services.dataset_review.orchestrator_pkg._helpers import ( extract_effective_filter_value, ) -logger = cast(Any, logger) +log = MarkerLogger("Orchestrator") # [DEF:DatasetReviewOrchestrator:Class] @@ -143,19 +144,19 @@ class DatasetReviewOrchestrator: normalized_environment_id = str(command.environment_id or "").strip() if not normalized_source_input: - logger.explore("Blocked dataset review session start due to empty source input") + log.explore("Blocked dataset review session start due to empty source input", error="source_input was empty") raise ValueError("source_input must be non-empty") if normalized_source_kind not in {"superset_link", "dataset_selection"}: - logger.explore("Blocked dataset review session start due to unsupported source kind", extra={"source_kind": normalized_source_kind}) + log.explore("Blocked dataset review session start due to unsupported source kind", payload={"source_kind": normalized_source_kind}, error="Unsupported source_kind") raise ValueError("source_kind must be 'superset_link' or 'dataset_selection'") environment = self.config_manager.get_environment(normalized_environment_id) if environment is None: - logger.explore("Blocked dataset review session start because environment was not found", extra={"environment_id": normalized_environment_id}) + log.explore("Blocked dataset review session start because environment was not found", payload={"environment_id": normalized_environment_id}, error="Environment not found in config") raise ValueError("Environment not found") - logger.reason("Starting dataset review session", extra={"user_id": command.user.id, "environment_id": normalized_environment_id, "source_kind": normalized_source_kind}) + log.reason("Starting dataset review session", payload={"user_id": command.user.id, "environment_id": normalized_environment_id, "source_kind": normalized_source_kind}) parsed_context: Optional[SupersetParsedContext] = None findings: List[ValidationFinding] = [] @@ -217,7 +218,7 @@ class DatasetReviewOrchestrator: parsed_context=parsed_context, dataset_ref=dataset_ref, ) - self.repository.event_logger.log_event( + self.repository.event_log.log_event( SessionEventPayload( session_id=persisted_session.session_id, actor_user_id=command.user.id, @@ -259,7 +260,7 @@ class DatasetReviewOrchestrator: self.repository.bump_session_version(persisted_session) self.repository.db.commit() self.repository.db.refresh(persisted_session) - self.repository.event_logger.log_event( + self.repository.event_log.log_event( SessionEventPayload( session_id=persisted_session.session_id, actor_user_id=command.user.id, @@ -270,9 +271,9 @@ class DatasetReviewOrchestrator: event_details={"task_id": active_task_id}, ) ) - logger.reason("Linked recovery task to started dataset review session", extra={"session_id": persisted_session.session_id, "task_id": active_task_id}) + log.reason("Linked recovery task to started dataset review session", payload={"session_id": persisted_session.session_id, "task_id": active_task_id}) - logger.reflect("Dataset review session start completed", extra={"session_id": persisted_session.session_id, "dataset_ref": persisted_session.dataset_ref, "readiness_state": persisted_session.readiness_state.value, "active_task_id": persisted_session.active_task_id, "finding_count": len(findings)}) + log.reflect("Dataset review session start completed", payload={"session_id": persisted_session.session_id, "dataset_ref": persisted_session.dataset_ref, "readiness_state": persisted_session.readiness_state.value, "active_task_id": persisted_session.active_task_id, "finding_count": len(findings)}) return StartSessionResult( session=persisted_session, parsed_context=parsed_context, @@ -293,7 +294,7 @@ class DatasetReviewOrchestrator: with belief_scope("DatasetReviewOrchestrator.prepare_launch_preview"): session = self.repository.load_session_detail(command.session_id, command.user.id) if session is None or session.user_id != command.user.id: - logger.explore("Preview preparation rejected because owned session was not found", extra={"session_id": command.session_id, "user_id": command.user.id}) + log.explore("Preview preparation rejected because owned session was not found", payload={"session_id": command.session_id, "user_id": command.user.id}, error="Session not found or access denied") raise ValueError("Session not found") if command.expected_version is not None: @@ -309,7 +310,7 @@ class DatasetReviewOrchestrator: execution_snapshot = build_execution_snapshot(session) preview_blockers = execution_snapshot["preview_blockers"] if preview_blockers: - logger.explore("Preview preparation blocked by incomplete execution context", extra={"session_id": session.session_id, "blocked_reasons": preview_blockers}) + log.explore("Preview preparation blocked by incomplete execution context", payload={"session_id": session.session_id, "blocked_reasons": preview_blockers}, error="Preview blockers present") raise ValueError("Preview blocked: " + "; ".join(preview_blockers)) adapter = SupersetCompilationAdapter(environment) @@ -344,7 +345,7 @@ class DatasetReviewOrchestrator: session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW self.repository.db.commit() self.repository.db.refresh(session) - self.repository.event_logger.log_event( + self.repository.event_log.log_event( SessionEventPayload( session_id=session.session_id, actor_user_id=command.user.id, @@ -356,7 +357,7 @@ class DatasetReviewOrchestrator: ) ) - logger.reflect("Superset preview preparation completed", extra={"session_id": session.session_id, "preview_id": persisted_preview.preview_id, "preview_status": persisted_preview.preview_status.value}) + log.reflect("Superset preview preparation completed", payload={"session_id": session.session_id, "preview_id": persisted_preview.preview_id, "preview_status": persisted_preview.preview_status.value}) return PreparePreviewResult(session=session, preview=persisted_preview, blocked_reasons=[]) # [/DEF:prepare_launch_preview:Function] @@ -374,7 +375,7 @@ class DatasetReviewOrchestrator: with belief_scope("DatasetReviewOrchestrator.launch_dataset"): session = self.repository.load_session_detail(command.session_id, command.user.id) if session is None or session.user_id != command.user.id: - logger.explore("Launch rejected because owned session was not found", extra={"session_id": command.session_id, "user_id": command.user.id}) + log.explore("Launch rejected because owned session was not found", payload={"session_id": command.session_id, "user_id": command.user.id}, error="Session not found or access denied") raise ValueError("Session not found") if command.expected_version is not None: @@ -391,7 +392,7 @@ class DatasetReviewOrchestrator: current_preview = get_latest_preview(session) launch_blockers_list = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=current_preview) if launch_blockers_list: - logger.explore("Launch gate blocked dataset execution", extra={"session_id": session.session_id, "blocked_reasons": launch_blockers_list}) + log.explore("Launch gate blocked dataset execution", payload={"session_id": session.session_id, "blocked_reasons": launch_blockers_list}, error="Launch blockers present") raise ValueError("Launch blocked: " + "; ".join(launch_blockers_list)) adapter = SupersetCompilationAdapter(environment) @@ -408,7 +409,7 @@ class DatasetReviewOrchestrator: launch_status = LaunchStatus.STARTED launch_error = None except Exception as exc: - logger.explore("SQL Lab launch failed after passing gates", extra={"session_id": session.session_id, "error": str(exc)}) + log.explore("SQL Lab launch failed after passing gates", payload={"session_id": session.session_id}, error=str(exc)) sql_lab_session_ref = "unavailable" launch_status = LaunchStatus.FAILED launch_error = str(exc) @@ -444,7 +445,7 @@ class DatasetReviewOrchestrator: session.recommended_action = RecommendedAction.EXPORT_OUTPUTS self.repository.db.commit() self.repository.db.refresh(session) - self.repository.event_logger.log_event( + self.repository.event_log.log_event( SessionEventPayload( session_id=session.session_id, actor_user_id=command.user.id, @@ -456,7 +457,7 @@ class DatasetReviewOrchestrator: ) ) - logger.reflect("Dataset launch orchestration completed with audited run context", extra={"session_id": session.session_id, "run_context_id": persisted_run_context.run_context_id, "launch_status": persisted_run_context.launch_status.value}) + log.reflect("Dataset launch orchestration completed with audited run context", payload={"session_id": session.session_id, "run_context_id": persisted_run_context.run_context_id, "launch_status": persisted_run_context.launch_status.value}) return LaunchDatasetResult(session=session, run_context=persisted_run_context, blocked_reasons=[]) # [/DEF:launch_dataset:Function] @@ -532,7 +533,7 @@ class DatasetReviewOrchestrator: caused_by_ref="dataset_template_variable_discovery_failed", ) ) - logger.explore("Template variable discovery failed during session bootstrap", extra={"session_id": session_record.session_id, "dataset_id": session_record.dataset_id, "error": str(exc)}) + log.explore("Template variable discovery failed during session bootstrap", payload={"session_id": session_record.session_id, "dataset_id": session_record.dataset_id}, error=str(exc)) filter_lookup = {str(f.filter_name or "").strip().lower(): f for f in imported_filters if str(f.filter_name or "").strip()} for tv in template_variables: @@ -575,7 +576,7 @@ class DatasetReviewOrchestrator: ) -> Optional[str]: session_record = cast(Any, session) if self.task_manager is None: - logger.reason("Dataset review session started without task manager; continuing synchronously", extra={"session_id": session_record.session_id}) + log.reason("Dataset review session started without task manager; continuing synchronously", payload={"session_id": session_record.session_id}) return None task_params: Dict[str, Any] = { @@ -592,13 +593,13 @@ class DatasetReviewOrchestrator: create_task = getattr(self.task_manager, "create_task", None) if create_task is None: - logger.explore("Task manager has no create_task method; skipping recovery enqueue") + log.explore("Task manager has no create_task method; skipping recovery enqueue", error="create_task method not found") return None try: task_object = create_task(plugin_id="dataset-review-recovery", params=task_params) except TypeError: - logger.explore("Recovery task enqueue skipped because task manager create_task contract is incompatible", extra={"session_id": session_record.session_id}) + log.explore("Recovery task enqueue skipped because task manager create_task contract is incompatible", payload={"session_id": session_record.session_id}, error="TypeError from create_task") return None task_id = getattr(task_object, "id", None) diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py index e01c7462..56cfb69b 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py @@ -14,7 +14,7 @@ import json from datetime import datetime from typing import Any, Dict, List, Optional, cast -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger from src.models.dataset_review import ( ApprovalState, CompiledPreview, @@ -38,8 +38,7 @@ from src.models.dataset_review import ( BusinessSummarySource, ) -logger = cast(Any, logger) - +log = MarkerLogger("OrchestratorHelpers") # [DEF:parse_dataset_selection:Function] # @COMPLEXITY: 2 diff --git a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py index 06cce284..4f6c9fb9 100644 --- a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py +++ b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py @@ -14,7 +14,8 @@ from typing import Any, List, Optional, cast from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.models.dataset_review import ( ClarificationQuestion, ClarificationSession, @@ -32,7 +33,7 @@ from src.models.dataset_review import ( ) from src.services.dataset_review.event_logger import SessionEventLogger -logger = cast(Any, logger) +log = MarkerLogger("SessionMutations") # [DEF:save_profile_and_findings:Function] @@ -57,7 +58,7 @@ def save_profile_and_findings( session = get_owned_session(session_id, user_id) if expected_version is not None: require_session_version(session, expected_version) - logger.reason("Persisting dataset profile and replacing validation findings", extra={"session_id": session_id, "user_id": user_id, "has_profile": bool(profile), "findings_count": len(findings)}) + log.reason("Persisting dataset profile and replacing validation findings", payload={"session_id": session_id, "user_id": user_id, "has_profile": bool(profile), "findings_count": len(findings)}) if profile: existing_profile = db.query(DatasetProfile).filter_by(session_id=session_id).first() @@ -71,7 +72,7 @@ def save_profile_and_findings( db.add(finding) commit_session_mutation(session, expected_version=expected_version) - logger.reflect("Dataset profile and validation findings committed", extra={"session_id": session.session_id, "user_id": user_id, "findings_count": len(findings)}) + log.reflect("Dataset profile and validation findings committed", payload={"session_id": session.session_id, "user_id": user_id, "findings_count": len(findings)}) from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository return session @@ -103,7 +104,7 @@ def save_recovery_state( session = get_owned_session(session_id, user_id) if expected_version is not None: require_session_version(session, expected_version) - logger.reason("Persisting dataset review recovery bootstrap state", extra={"session_id": session_id, "user_id": user_id, "imported_filters_count": len(imported_filters), "template_variables_count": len(template_variables), "execution_mappings_count": len(execution_mappings)}) + log.reason("Persisting dataset review recovery bootstrap state", payload={"session_id": session_id, "user_id": user_id, "imported_filters_count": len(imported_filters), "template_variables_count": len(template_variables), "execution_mappings_count": len(execution_mappings)}) db.query(ExecutionMapping).filter(ExecutionMapping.session_id == session_id).delete() db.query(TemplateVariable).filter(TemplateVariable.session_id == session_id).delete() @@ -121,7 +122,7 @@ def save_recovery_state( db.add(em) commit_session_mutation(session, expected_version=expected_version) - logger.reflect("Dataset review recovery bootstrap state committed", extra={"session_id": session.session_id, "user_id": user_id}) + log.reflect("Dataset review recovery bootstrap state committed", payload={"session_id": session.session_id, "user_id": user_id}) return load_session_detail_fn(session_id, user_id) @@ -149,7 +150,7 @@ def save_preview( session_record = cast(Any, session) if expected_version is not None: require_session_version(session, expected_version) - logger.reason("Persisting compiled preview and staling previous preview snapshots", extra={"session_id": session_id, "user_id": user_id}) + log.reason("Persisting compiled preview and staling previous preview snapshots", payload={"session_id": session_id, "user_id": user_id}) db.query(CompiledPreview).filter(CompiledPreview.session_id == session_id).update({"preview_status": "stale"}) db.add(preview) @@ -157,7 +158,7 @@ def save_preview( session_record.last_preview_id = preview.preview_id commit_session_mutation(session, refresh_targets=[preview], expected_version=expected_version) - logger.reflect("Compiled preview committed as latest session preview", extra={"session_id": session.session_id, "preview_id": preview.preview_id}) + log.reflect("Compiled preview committed as latest session preview", payload={"session_id": session.session_id, "preview_id": preview.preview_id}) return preview @@ -185,14 +186,14 @@ def save_run_context( session_record = cast(Any, session) if expected_version is not None: require_session_version(session, expected_version) - logger.reason("Persisting dataset run context audit snapshot", extra={"session_id": session_id, "user_id": user_id}) + log.reason("Persisting dataset run context audit snapshot", payload={"session_id": session_id, "user_id": user_id}) db.add(run_context) db.flush() session_record.last_run_context_id = run_context.run_context_id commit_session_mutation(session, refresh_targets=[run_context], expected_version=expected_version) - logger.reflect("Dataset run context committed as latest launch snapshot", extra={"session_id": session.session_id, "run_context_id": run_context.run_context_id}) + log.reflect("Dataset run context committed as latest launch snapshot", payload={"session_id": session.session_id, "run_context_id": run_context.run_context_id}) return run_context diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index 6e0ea5f7..6ba34e58 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -16,10 +16,12 @@ # @REJECTED: Keeping all repository operations in one file because it exceeded the fractal limit. from datetime import datetime -from typing import Any, Optional, List, cast +from typing import Any, Optional, List from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload from sqlalchemy.orm.exc import StaleDataError +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.models.dataset_review import ( ClarificationQuestion, ClarificationSession, @@ -35,10 +37,9 @@ from src.models.dataset_review import ( SessionEvent, TemplateVariable, ) -from src.core.logger import belief_scope, logger from src.services.dataset_review.event_logger import SessionEventLogger -logger = cast(Any, logger) +log = MarkerLogger("SessionRepository") # [DEF:DatasetReviewSessionVersionConflictError:Class] @@ -84,16 +85,16 @@ class DatasetReviewSessionRepository: # @POST: returns the owned session or raises a deterministic access error. def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.get_owned_session"): - logger.reason("Resolving owner-scoped dataset review session", extra={"session_id": session_id, "user_id": user_id}) + log.reason("Resolving owner-scoped dataset review session", payload={"session_id": session_id, "user_id": user_id}) session = ( self.db.query(DatasetReviewSession) .filter(DatasetReviewSession.session_id == session_id, DatasetReviewSession.user_id == user_id) .first() ) if not session: - logger.explore("Owner-scoped dataset review session lookup failed", extra={"session_id": session_id, "user_id": user_id}) + log.explore("Owner-scoped dataset review session lookup failed", payload={"session_id": session_id, "user_id": user_id}, error="Session not found or access denied") raise ValueError("Session not found or access denied") - logger.reflect("Owner-scoped dataset review session resolved", extra={"session_id": session.session_id}) + log.reflect("Owner-scoped dataset review session resolved", payload={"session_id": session.session_id}) return session # [/DEF:get_owned_session:Function] @@ -104,11 +105,11 @@ class DatasetReviewSessionRepository: # @POST: session is committed, refreshed, and returned with persisted identifiers. def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.create_session"): - logger.reason("Persisting dataset review session shell", extra={"user_id": session.user_id, "environment_id": session.environment_id}) + log.reason("Persisting dataset review session shell", payload={"user_id": session.user_id, "environment_id": session.environment_id}) self.db.add(session) self.db.commit() self.db.refresh(session) - logger.reflect("Dataset review session shell persisted", extra={"session_id": session.session_id}) + log.reflect("Dataset review session shell persisted", payload={"session_id": session.session_id}) return session # [/DEF:create_sess:Function] @@ -120,11 +121,11 @@ class DatasetReviewSessionRepository: def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.require_session_version"): actual_version = int(getattr(session, "version", 0) or 0) - logger.reason("Checking optimistic-lock version", extra={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}) + log.reason("Checking optimistic-lock version", payload={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}) if actual_version != expected_version: - logger.explore("Rejected mutation due to stale session version", extra={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}) + log.explore("Rejected mutation due to stale session version", payload={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}, error="Optimistic lock version mismatch") raise DatasetReviewSessionVersionConflictError(str(session.session_id), expected_version, actual_version) - logger.reflect("Optimistic-lock version accepted", extra={"session_id": session.session_id, "version": actual_version}) + log.reflect("Optimistic-lock version accepted", payload={"session_id": session.session_id, "version": actual_version}) return session # [/DEF:require_session_version:Function] @@ -138,7 +139,7 @@ class DatasetReviewSessionRepository: next_version = int(getattr(session, "version", 0) or 0) + 1 setattr(session, "version", next_version) session.last_activity_at = datetime.utcnow() - logger.reflect("Prepared incremented session version", extra={"session_id": session.session_id, "version": next_version}) + log.reflect("Prepared incremented session version", payload={"session_id": session.session_id, "version": next_version}) return next_version # [/DEF:bump_session_version:Function] @@ -152,7 +153,7 @@ class DatasetReviewSessionRepository: ) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.commit_session_mutation"): observed_version = int(expected_version if expected_version is not None else getattr(session, "version", 0) or 0) - logger.reason("Committing session mutation with optimistic lock", extra={"session_id": session.session_id, "observed_version": observed_version}) + log.reason("Committing session mutation with optimistic lock", payload={"session_id": session.session_id, "observed_version": observed_version}) self.bump_session_version(session) try: self.db.commit() @@ -160,12 +161,12 @@ class DatasetReviewSessionRepository: self.db.rollback() actual_version_row = self.db.query(DatasetReviewSession.version).filter(DatasetReviewSession.session_id == session.session_id).first() actual_version = int(actual_version_row[0] or 0) if actual_version_row else 0 - logger.explore("Session commit rejected by optimistic lock", extra={"session_id": session.session_id, "expected_version": observed_version, "actual_version": actual_version}) + log.explore("Session commit rejected by optimistic lock", payload={"session_id": session.session_id, "expected_version": observed_version, "actual_version": actual_version}, error="StaleDataError on session commit") raise DatasetReviewSessionVersionConflictError(session.session_id, observed_version, actual_version) from exc self.db.refresh(session) for target in refresh_targets or []: self.db.refresh(target) - logger.reflect("Session mutation committed", extra={"session_id": session.session_id, "version": getattr(session, "version", None)}) + log.reflect("Session mutation committed", payload={"session_id": session.session_id, "version": getattr(session, "version", None)}) return session # [/DEF:commit_session_mutation:Function] @@ -176,7 +177,7 @@ class DatasetReviewSessionRepository: # @POST: Returns SessionDetail with all fields populated or None. def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.load_session_detail"): - logger.reason("Loading dataset review session detail", extra={"session_id": session_id, "user_id": user_id}) + log.reason("Loading dataset review session detail", payload={"session_id": session_id, "user_id": user_id}) session = ( self.db.query(DatasetReviewSession) .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id) @@ -199,7 +200,7 @@ class DatasetReviewSessionRepository: .filter(or_(DatasetReviewSession.user_id == user_id, SessionCollaborator.user_id == user_id)) .first() ) - logger.reflect("Session detail lookup completed", extra={"session_id": session_id, "found": bool(session)}) + log.reflect("Session detail lookup completed", payload={"session_id": session_id, "found": bool(session)}) return session # [/DEF:load_detail:Function] @@ -269,14 +270,14 @@ class DatasetReviewSessionRepository: # @PURPOSE: List review sessions owned by a specific user ordered by most recent update. def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.list_sessions_for_user"): - logger.reason("Listing dataset review sessions for owner scope", extra={"user_id": user_id}) + log.reason("Listing dataset review sessions for owner scope", payload={"user_id": user_id}) sessions = ( self.db.query(DatasetReviewSession) .filter(DatasetReviewSession.user_id == user_id) .order_by(DatasetReviewSession.updated_at.desc()) .all() ) - logger.reflect("Session list assembled", extra={"user_id": user_id, "session_count": len(sessions)}) + log.reflect("Session list assembled", payload={"user_id": user_id, "session_count": len(sessions)}) return sessions # [/DEF:list_user_sess:Function] diff --git a/backend/src/services/dataset_review/semantic_resolver.py b/backend/src/services/dataset_review/semantic_resolver.py index b8e924ef..ad103266 100644 --- a/backend/src/services/dataset_review/semantic_resolver.py +++ b/backend/src/services/dataset_review/semantic_resolver.py @@ -19,7 +19,8 @@ from dataclasses import dataclass, field from difflib import SequenceMatcher from typing import Any, Dict, Iterable, List, Mapping, Optional -from src.core.logger import belief_scope, logger +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope from src.models.dataset_review import ( CandidateMatchType, CandidateStatus, @@ -28,6 +29,8 @@ from src.models.dataset_review import ( ) # [/DEF:imports:Block] +log = MarkerLogger("SemanticSourceResolver") + # [DEF:DictionaryResolutionResult:Class] # @COMPLEXITY: 2 @@ -76,19 +79,16 @@ class SemanticSourceResolver: dictionary_rows = source_payload.get("rows") if not source_ref: - logger.explore("Dictionary semantic source is missing source_ref") + log.explore("Dictionary semantic source is missing source_ref", error="No source_ref in dictionary source") raise ValueError("Dictionary semantic source must include source_ref") if not isinstance(dictionary_rows, list) or not dictionary_rows: - logger.explore( - "Dictionary semantic source has no usable rows", - extra={"source_ref": source_ref}, - ) + log.explore("Dictionary semantic source has no usable rows", error="Dictionary rows is empty or not a list", payload={"source_ref": source_ref}) raise ValueError("Dictionary semantic source must include non-empty rows") - logger.reason( + log.reason( "Resolving semantics from trusted dictionary source", - extra={"source_ref": source_ref, "row_count": len(dictionary_rows)}, + payload={"source_ref": source_ref, "row_count": len(dictionary_rows)}, ) normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)] @@ -108,9 +108,9 @@ class SemanticSourceResolver: is_locked = bool(raw_field.get("is_locked")) if is_locked: - logger.reason( + log.reason( "Preserving manual lock during dictionary resolution", - extra={"field_name": field_name}, + payload={"field_name": field_name}, ) resolved_fields.append( { @@ -130,9 +130,9 @@ class SemanticSourceResolver: candidates: List[Dict[str, Any]] = [] if exact_match is not None: - logger.reason( + log.reason( "Resolved exact dictionary match", - extra={"field_name": field_name, "source_ref": source_ref}, + payload={"field_name": field_name, "source_ref": source_ref}, ) candidates.append( self._build_candidate_payload( @@ -168,10 +168,7 @@ class SemanticSourceResolver: "status": "unresolved", } ) - logger.explore( - "No trusted dictionary match found for field", - extra={"field_name": field_name, "source_ref": source_ref}, - ) + log.explore("No trusted dictionary match found for field", error="No dictionary match found for field", payload={"field_name": field_name, "source_ref": source_ref}) continue ranked_candidates = self.rank_candidates(candidates) @@ -203,9 +200,9 @@ class SemanticSourceResolver: unresolved_fields=unresolved_fields, partial_recovery=bool(unresolved_fields), ) - logger.reflect( + log.reflect( "Dictionary resolution completed", - extra={ + payload={ "source_ref": source_ref, "resolved_fields": len(resolved_fields), "unresolved_fields": len(unresolved_fields), @@ -278,10 +275,7 @@ class SemanticSourceResolver: source_id = str(source.source_id or "").strip() source_version = str(source.source_version or "").strip() if not source_id or not source_version: - logger.explore( - "Semantic source version propagation rejected due to incomplete source metadata", - extra={"source_id": source_id, "source_version": source_version}, - ) + log.explore("Semantic source version propagation rejected due to incomplete source metadata", error="Source metadata incomplete (missing source_id or source_version)", payload={"source_id": source_id, "source_version": source_version}) raise ValueError("Semantic source must provide source_id and source_version") propagated = 0 @@ -300,9 +294,9 @@ class SemanticSourceResolver: field.has_conflict = bool(getattr(field, "has_conflict", False)) propagated += 1 - logger.reflect( + log.reflect( "Semantic source version propagation completed", - extra={ + payload={ "source_id": source_id, "source_version": source_version, "propagated": propagated, diff --git a/backend/src/services/git/_base.py b/backend/src/services/git/_base.py index a414725d..6075a9f3 100644 --- a/backend/src/services/git/_base.py +++ b/backend/src/services/git/_base.py @@ -16,7 +16,10 @@ from typing import Any, Dict, List, Optional from git import Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError from fastapi import HTTPException -from src.core.logger import logger, belief_scope +from src.core.logger import belief_scope +from src.core.cot_logger import MarkerLogger + +log = MarkerLogger("GitBase") from src.models.git import GitRepository from src.models.config import AppConfigRecord from src.core.database import SessionLocal @@ -51,9 +54,7 @@ class GitServiceBase: try: base.mkdir(parents=True, exist_ok=True) except (PermissionError, OSError) as e: - logger.warning( - f"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}" - ) + log.explore("Cannot create Git repositories base path", payload={"base_path": self.base_path}, error=str(e)) raise ValueError(f"Cannot create Git repositories base path: {self.base_path}. {e}") # [/DEF:_ensure_base_path_exists:Function] @@ -88,7 +89,7 @@ class GitServiceBase: return str(repo_root.resolve()) return str((root / repo_root).resolve()) except Exception as e: - logger.warning(f"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}") + log.explore("Falling back to default base path", error=str(e)) return fallback_path # [/DEF:_resolve_base_path:Function] @@ -122,7 +123,7 @@ class GitServiceBase: finally: session.close() except Exception as e: - logger.warning(f"[_update_repo_local_path][Coherence:Failed] {e}") + log.explore("Failed to update repo local path", error=str(e)) # [/DEF:_update_repo_local_path:Function] # [DEF:_migrate_repo_directory:Function] @@ -136,7 +137,7 @@ class GitServiceBase: if source_abs == target_abs: return source_abs if os.path.exists(target_abs): - logger.warning(f"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}") + log.explore(f"Target already exists, keeping source path: {target_abs}", error="Target path exists") return source_abs Path(target_abs).parent.mkdir(parents=True, exist_ok=True) try: @@ -144,9 +145,7 @@ class GitServiceBase: except OSError: shutil.move(source_abs, target_abs) self._update_repo_local_path(dashboard_id, target_abs) - logger.info( - f"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}" - ) + log.reflect(f"Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}") return target_abs # [/DEF:_migrate_repo_directory:Function] @@ -187,7 +186,7 @@ class GitServiceBase: return self._migrate_repo_directory(dashboard_id, db_path, target_path) return db_path except Exception as e: - logger.warning(f"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}") + log.explore("Could not resolve local_path from DB", error=str(e)) legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id)) if os.path.exists(legacy_id_path) and not os.path.exists(target_path): return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path) @@ -213,11 +212,11 @@ class GitServiceBase: else: auth_url = remote_url if os.path.exists(repo_path): - logger.info(f"[init_repo][Action] Opening existing repo at {repo_path}") + log.reason(f"Opening existing repo at {repo_path}") try: repo = Repo(repo_path) except (InvalidGitRepositoryError, NoSuchPathError): - logger.warning(f"[init_repo][Action] Existing path is not a Git repository, recreating: {repo_path}") + log.explore(f"Existing path is not a Git repository, recreating: {repo_path}", error="Not a git repository") stale_path = Path(repo_path) if stale_path.exists(): shutil.rmtree(stale_path, ignore_errors=True) @@ -229,7 +228,7 @@ class GitServiceBase: repo = Repo.clone_from(auth_url, repo_path) self._ensure_gitflow_branches(repo, dashboard_id) return repo - logger.info(f"[init_repo][Action] Cloning {remote_url} to {repo_path}") + log.reason(f"Cloning {remote_url} to {repo_path}") repo = Repo.clone_from(auth_url, repo_path) self._ensure_gitflow_branches(repo, dashboard_id) return repo @@ -271,7 +270,7 @@ class GitServiceBase: raise except Exception as e: session.rollback() - logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}") + log.explore(f"Failed to delete repository for dashboard {dashboard_id}", error=str(e)) raise HTTPException(status_code=500, detail=f"Failed to delete repository: {str(e)}") finally: session.close() @@ -286,12 +285,12 @@ class GitServiceBase: with belief_scope("GitService.get_repo"): repo_path = self._get_repo_path(dashboard_id) if not os.path.exists(repo_path): - logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist") + log.explore(f"Repository for dashboard {dashboard_id} does not exist", error="Repository not found") raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found") try: return Repo(repo_path) except Exception as e: - logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}") + log.explore(f"Failed to open repository at {repo_path}", error=str(e)) raise HTTPException(status_code=500, detail="Failed to open local Git repository") # [/DEF:get_repo:Function] @@ -310,9 +309,9 @@ class GitServiceBase: with repo.config_writer(config_level="repository") as config_writer: config_writer.set_value("user", "name", normalized_username) config_writer.set_value("user", "email", normalized_email) - logger.info("[configure_identity][Action] Applied repository-local git identity for dashboard %s", dashboard_id) + log.reason(f"Applied repository-local git identity for dashboard {dashboard_id}") except Exception as e: - logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}") + log.explore("Failed to configure git identity", error=str(e)) raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {str(e)}") # [/DEF:configure_identity:Function] # [/DEF:GitServiceBase:Class] diff --git a/backend/src/services/git/_branch.py b/backend/src/services/git/_branch.py index 2122769d..7127d6df 100644 --- a/backend/src/services/git/_branch.py +++ b/backend/src/services/git/_branch.py @@ -11,7 +11,10 @@ from datetime import datetime from git import Repo from git.exc import GitCommandError from fastapi import HTTPException -from src.core.logger import logger, belief_scope +from src.core.logger import belief_scope +from src.core.cot_logger import MarkerLogger + +log = MarkerLogger("GitBranch") # [DEF:GitServiceBranchMixin:Class] @@ -34,26 +37,20 @@ class GitServiceBranchMixin: if "main" in local_heads: base_commit = local_heads["main"].commit if base_commit is None: - logger.warning( - f"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits" - ) + log.explore("Branch bootstrap skipped - no commits", payload={"dashboard_id": dashboard_id}, error="Repository has no initial commit to branch from") return if "main" not in local_heads: local_heads["main"] = repo.create_head("main", base_commit) - logger.info(f"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}") + log.reason("Created local branch main", payload={"dashboard_id": dashboard_id}) for branch_name in ("dev", "preprod"): if branch_name in local_heads: continue local_heads[branch_name] = repo.create_head(branch_name, local_heads["main"].commit) - logger.info( - f"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}" - ) + log.reason("Created local branch", payload={"branch_name": branch_name, "dashboard_id": dashboard_id}) try: origin = repo.remote(name="origin") except ValueError: - logger.info( - f"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation" - ) + log.reason("Remote origin not configured; skipping remote branch creation", payload={"dashboard_id": dashboard_id}) return remote_branch_names = set() try: @@ -63,26 +60,24 @@ class GitServiceBranchMixin: if remote_head: remote_branch_names.add(str(remote_head)) except Exception as e: - logger.warning(f"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}") + log.explore("Failed to fetch origin refs", error=str(e)) for branch_name in required_branches: if branch_name in remote_branch_names: continue try: origin.push(refspec=f"{branch_name}:{branch_name}") - logger.info( - f"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}" - ) + log.reason("Pushed branch to origin", payload={"branch_name": branch_name, "dashboard_id": dashboard_id}) except Exception as e: - logger.error(f"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}") + log.explore("Failed to push branch to origin", error=str(e)) raise HTTPException( status_code=500, detail=f"Failed to create default branch '{branch_name}' on remote: {str(e)}", ) try: repo.git.checkout("dev") - logger.info(f"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}") + log.reason("Checked out default branch dev", payload={"dashboard_id": dashboard_id}) except Exception as e: - logger.warning(f"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}") + log.explore("Could not checkout dev branch", payload={"dashboard_id": dashboard_id}, error=str(e)) # [/DEF:_ensure_gitflow_branches:Function] # [DEF:list_branches:Function] @@ -93,7 +88,7 @@ class GitServiceBranchMixin: def list_branches(self, dashboard_id: int) -> List[dict]: with belief_scope("GitService.list_branches"): repo = self.get_repo(dashboard_id) - logger.info(f"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}") + log.reason("Listing branches", payload={"dashboard_id": dashboard_id}) branches = [] for ref in repo.refs: try: @@ -107,7 +102,7 @@ class GitServiceBranchMixin: "last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow() }) except Exception as e: - logger.warning(f"[list_branches][Action] Skipping ref {ref}: {e}") + log.explore("Skipping ref", payload={"ref": str(ref)}, error=str(e)) try: active_name = repo.active_branch.name if not any(b['name'] == active_name for b in branches): @@ -116,7 +111,7 @@ class GitServiceBranchMixin: "is_remote": False, "last_updated": datetime.utcnow() }) except Exception as e: - logger.warning(f"[list_branches][Action] Could not determine active branch: {e}") + log.explore("Could not determine active branch", error=str(e)) if not branches: branches.append({ "name": "dev", "commit_hash": "0000000", @@ -134,9 +129,9 @@ class GitServiceBranchMixin: def create_branch(self, dashboard_id: int, name: str, from_branch: str = "main"): with belief_scope("GitService.create_branch"): repo = self.get_repo(dashboard_id) - logger.info(f"[create_branch][Action] Creating branch {name} from {from_branch}") + log.reason("Creating branch", payload={"name": name, "from_branch": from_branch}) if not repo.heads and not repo.remotes: - logger.warning("[create_branch][Action] Repository is empty. Creating initial commit to enable branching.") + log.explore("Repository is empty; creating initial commit to enable branching", error="No branches or remotes exist; initializing from scratch") readme_path = os.path.join(repo.working_dir, "README.md") if not os.path.exists(readme_path): with open(readme_path, "w") as f: @@ -146,13 +141,13 @@ class GitServiceBranchMixin: try: repo.commit(from_branch) except Exception: - logger.warning(f"[create_branch][Action] Source branch {from_branch} not found, using HEAD") + log.explore("Source branch not found, using HEAD", payload={"from_branch": from_branch}, error=f"Branch '{from_branch}' does not exist, falling back to HEAD") from_branch = repo.head try: new_branch = repo.create_head(name, from_branch) return new_branch except Exception as e: - logger.error(f"[create_branch][Coherence:Failed] {e}") + log.explore("Failed to create branch", error=str(e)) raise # [/DEF:create_branch:Function] @@ -163,7 +158,7 @@ class GitServiceBranchMixin: def checkout_branch(self, dashboard_id: int, name: str): with belief_scope("GitService.checkout_branch"): repo = self.get_repo(dashboard_id) - logger.info(f"[checkout_branch][Action] Checking out branch {name}") + log.reason("Checking out branch", payload={"name": name}) repo.git.checkout(name) # [/DEF:checkout_branch:Function] @@ -177,16 +172,16 @@ class GitServiceBranchMixin: with belief_scope("GitService.commit_changes"): repo = self.get_repo(dashboard_id) if not repo.is_dirty(untracked_files=True) and not files: - logger.info(f"[commit_changes][Action] No changes to commit for dashboard {dashboard_id}") + log.reason("No changes to commit", payload={"dashboard_id": dashboard_id}) return if files: - logger.info(f"[commit_changes][Action] Staging files: {files}") + log.reason("Staging files", payload={"files": files}) repo.index.add(files) else: - logger.info("[commit_changes][Action] Staging all changes") + log.reason("Staging all changes") repo.git.add(A=True) repo.index.commit(message) - logger.info(f"[commit_changes][Coherence:OK] Committed changes with message: {message}") + log.reflect("Committed changes", payload={"message": message}) # [/DEF:commit_changes:Function] # [/DEF:GitServiceBranchMixin:Class] # [/DEF:GitServiceBranchMixin:Module] diff --git a/backend/src/services/git/_gitea.py b/backend/src/services/git/_gitea.py index 04fb90d3..e3f87363 100644 --- a/backend/src/services/git/_gitea.py +++ b/backend/src/services/git/_gitea.py @@ -9,7 +9,10 @@ import httpx from typing import Any, Dict, List, Optional from urllib.parse import quote from fastapi import HTTPException -from src.core.logger import logger, belief_scope +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope + +log = MarkerLogger("GitGitea") from src.models.git import GitProvider @@ -26,13 +29,13 @@ class GitServiceGiteaMixin: async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool: with belief_scope("GitService.test_connection"): if ".local" in url or "localhost" in url: - logger.info("[test_connection][Action] Local/Offline mode detected for URL") + log.reason("Local/Offline mode detected for URL") return True if not url.startswith(('http://', 'https://')): - logger.error(f"[test_connection][Coherence:Failed] Invalid URL protocol: {url}") + log.explore(f"Invalid URL protocol: {url}", error="Invalid URL protocol") return False if not pat or not pat.strip(): - logger.error("[test_connection][Coherence:Failed] Git PAT is missing or empty") + log.explore("Git PAT is missing or empty", error="Git PAT is missing or empty") return False pat = pat.strip() try: @@ -52,10 +55,10 @@ class GitServiceGiteaMixin: else: return False if resp.status_code != 200: - logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}") + log.explore(f"Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}", error="Git connection test failed") return resp.status_code == 200 except Exception as e: - logger.error(f"[test_connection][Coherence:Failed] Error testing git connection: {e}") + log.explore(f"Error testing git connection: {e}", error=str(e)) return False # [/DEF:test_connection:Function] @@ -91,7 +94,7 @@ class GitServiceGiteaMixin: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.request(method=method, url=url, headers=headers, json=payload) except Exception as e: - logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}") + log.explore(f"Network error: {e}", error=str(e)) raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {str(e)}") if response.status_code >= 400: detail = response.text @@ -100,7 +103,7 @@ class GitServiceGiteaMixin: detail = parsed.get("message") or parsed.get("error") or detail except Exception: pass - logger.error(f"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}") + log.explore(f"Gitea API error ({endpoint})", payload={"method": method, "status": response.status_code}, error=detail) raise HTTPException(status_code=response.status_code, detail=f"Gitea API error: {detail}") if response.status_code == 204: return None @@ -223,10 +226,7 @@ class GitServiceGiteaMixin: exc.status_code == 404 and fallback_url and fallback_url != normalized_primary ) if should_retry_with_fallback: - logger.warning( - "[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s", - fallback_url, - ) + log.explore(f"Primary Gitea URL not found, retrying with remote host: {fallback_url}", error=fallback_url) active_server_url = fallback_url try: data = await self._gitea_request("POST", active_server_url, pat, endpoint, payload=req_payload) diff --git a/backend/src/services/git/_status.py b/backend/src/services/git/_status.py index 303e0460..dff3801f 100644 --- a/backend/src/services/git/_status.py +++ b/backend/src/services/git/_status.py @@ -8,7 +8,10 @@ from typing import Any, Dict, List from datetime import datetime from git import Repo -from src.core.logger import logger, belief_scope +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope + +log = MarkerLogger("GitStatus") # [DEF:GitServiceStatusMixin:Class] @@ -31,7 +34,7 @@ class GitServiceStatusMixin: try: output = repo.git.status("--porcelain") except Exception: - logger.warning("[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed") + log.explore("git status --porcelain failed", error="git status --porcelain command failed") return staged, modified, untracked for line in output.split("\n"): if not line: @@ -156,7 +159,7 @@ class GitServiceStatusMixin: "files_changed": list(commit.stats.files.keys()) }) except Exception as e: - logger.warning(f"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}") + log.explore(f"Could not retrieve commit history for dashboard {dashboard_id}: {e}", error=str(e)) return [] return commits # [/DEF:get_commit_history:Function] diff --git a/backend/src/services/git/_sync.py b/backend/src/services/git/_sync.py index 4a340990..6a3c7975 100644 --- a/backend/src/services/git/_sync.py +++ b/backend/src/services/git/_sync.py @@ -12,7 +12,10 @@ from git import Repo from git.exc import GitCommandError from fastapi import HTTPException from src.core.database import SessionLocal -from src.core.logger import logger, belief_scope +from src.core.logger import belief_scope +from src.core.cot_logger import MarkerLogger + +log = MarkerLogger("GitSync") from src.models.git import GitRepository, GitServerConfig @@ -28,12 +31,12 @@ class GitServiceSyncMixin: with belief_scope("GitService.push_changes"): repo = self.get_repo(dashboard_id) if not repo.heads: - logger.warning(f"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}") + log.explore("No local branches to push", payload={"dashboard_id": dashboard_id}, error="Repository has no local branch heads to push") return try: origin = repo.remote(name='origin') except ValueError: - logger.error(f"[push_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") + log.explore("Remote 'origin' not found", payload={"dashboard_id": dashboard_id}, error="Git remote 'origin' is not configured for this repository") raise HTTPException(status_code=400, detail="Remote 'origin' not configured") try: origin_urls = list(origin.urls) @@ -63,10 +66,7 @@ class GitServiceSyncMixin: finally: session.close() except Exception as diag_error: - logger.warning( - "[push_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", - dashboard_id, diag_error, - ) + log.explore("Failed to load repository binding diagnostics", payload={"dashboard_id": dashboard_id}, error=str(diag_error)) realigned_origin_url = self._align_origin_host_with_config( dashboard_id=dashboard_id, origin=origin, @@ -78,13 +78,17 @@ class GitServiceSyncMixin: origin_urls = list(origin.urls) except Exception: origin_urls = [] - logger.info( - "[push_changes][Action] Push diagnostics dashboard=%s config_id=%s config_url=%s binding_remote_url=%s origin_urls=%s origin_realigned=%s", - dashboard_id, binding_config_id, binding_config_url, binding_remote_url, origin_urls, bool(realigned_origin_url), + log.reason( + "Push diagnostics", + payload={ + "dashboard_id": dashboard_id, "config_id": binding_config_id, + "config_url": binding_config_url, "binding_remote_url": binding_remote_url, + "origin_urls": origin_urls, "origin_realigned": bool(realigned_origin_url), + }, ) try: current_branch = repo.active_branch - logger.info(f"[push_changes][Action] Pushing branch {current_branch.name} to origin") + log.reason("Pushing branch", payload={"branch": current_branch.name, "origin": True}) tracking_branch = None try: tracking_branch = current_branch.tracking_branch() @@ -96,7 +100,7 @@ class GitServiceSyncMixin: push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}') for info in push_info: if info.flags & info.ERROR: - logger.error(f"[push_changes][Coherence:Failed] Error pushing ref {info.remote_ref_string}: {info.summary}") + log.explore("Error pushing ref", payload={"ref": info.remote_ref_string, "summary": info.summary}, error=f"Git push error: {info.summary}") raise Exception(f"Git push error for {info.remote_ref_string}: {info.summary}") except GitCommandError as e: details = str(e) @@ -106,10 +110,10 @@ class GitServiceSyncMixin: status_code=409, detail="Push rejected: remote branch contains newer commits. Run Pull first, resolve conflicts if any, then push again.", ) - logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") + log.explore("Failed to push changes", error=str(e)) raise HTTPException(status_code=500, detail=f"Git push failed: {details}") except Exception as e: - logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") + log.explore("Failed to push changes", error=str(e)) raise HTTPException(status_code=500, detail=f"Git push failed: {str(e)}") # [/DEF:push_changes:Function] @@ -123,11 +127,14 @@ class GitServiceSyncMixin: merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") if os.path.exists(merge_head_path): payload = self._build_unfinished_merge_payload(repo) - logger.warning( - "[pull_changes][Action] Unfinished merge detected for dashboard %s (repo_path=%s git_dir=%s branch=%s merge_head=%s merge_msg=%s)", - dashboard_id, payload["repository_path"], payload["git_dir"], - payload["current_branch"], payload["merge_head"], payload["merge_message_preview"], - ) + log.explore("Unfinished merge detected", error="Unfinished merge state found", + payload={ + "dashboard_id": dashboard_id, + "repo_path": payload["repository_path"], + "git_dir": payload["git_dir"], + "branch": payload["current_branch"], + "merge_head": payload["merge_head"], + }) raise HTTPException(status_code=409, detail=payload) try: origin = repo.remote(name='origin') @@ -136,26 +143,36 @@ class GitServiceSyncMixin: origin_urls = list(origin.urls) except Exception: origin_urls = [] - logger.info( - "[pull_changes][Action] Pull diagnostics dashboard=%s repo_path=%s branch=%s origin_urls=%s", - dashboard_id, repo.working_tree_dir, current_branch, origin_urls, + log.reason( + "Pull diagnostics", + payload={ + "dashboard_id": dashboard_id, + "repo_path": repo.working_tree_dir, + "branch": current_branch, + "origin_urls": origin_urls, + }, ) origin.fetch(prune=True) remote_ref = f"origin/{current_branch}" has_remote_branch = any(ref.name == remote_ref for ref in repo.refs) - logger.info( - "[pull_changes][Action] Pull remote branch check dashboard=%s branch=%s remote_ref=%s exists=%s", - dashboard_id, current_branch, remote_ref, has_remote_branch, + log.reason( + "Pull remote branch check", + payload={ + "dashboard_id": dashboard_id, + "branch": current_branch, + "remote_ref": remote_ref, + "exists": has_remote_branch, + }, ) if not has_remote_branch: raise HTTPException( status_code=409, detail=f"Remote branch '{current_branch}' does not exist yet. Push this branch first.", ) - logger.info(f"[pull_changes][Action] Pulling changes from origin/{current_branch}") + log.reason("Pulling changes", payload={"branch": current_branch}) repo.git.pull("--no-rebase", "origin", current_branch) except ValueError: - logger.error(f"[pull_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") + log.explore("Remote 'origin' not found", payload={"dashboard_id": dashboard_id}, error="Git remote 'origin' is not configured for this repository") raise HTTPException(status_code=400, detail="Remote 'origin' not configured") except GitCommandError as e: details = str(e) @@ -165,12 +182,12 @@ class GitServiceSyncMixin: status_code=409, detail="Pull requires conflict resolution. Resolve conflicts in repository and repeat operation.", ) - logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") + log.explore("Failed to pull changes", error=str(e)) raise HTTPException(status_code=500, detail=f"Git pull failed: {details}") except HTTPException: raise except Exception as e: - logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") + log.explore("Failed to pull changes", error=str(e)) raise HTTPException(status_code=500, detail=f"Git pull failed: {str(e)}") # [/DEF:pull_changes:Function] # [/DEF:GitServiceSyncMixin:Class] diff --git a/backend/src/services/git/_url.py b/backend/src/services/git/_url.py index 5d0ec135..3875087e 100644 --- a/backend/src/services/git/_url.py +++ b/backend/src/services/git/_url.py @@ -12,7 +12,10 @@ from typing import Any, Dict, List, Optional from urllib.parse import quote, urlparse from fastapi import HTTPException from src.core.database import SessionLocal -from src.core.logger import logger, belief_scope +from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope + +log = MarkerLogger("GitUrl") from src.models.git import GitRepository, GitServerConfig # [DEF:GitServiceUrlMixin:Class] @@ -118,17 +121,11 @@ class GitServiceUrlMixin: aligned_url = self._replace_host_in_url(source_origin_url, config_url) if not aligned_url: return None - logger.warning( - "[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url", - dashboard_id, config_host, origin_host, - ) + log.explore(f"Host mismatch for dashboard {dashboard_id}: config_host={config_host} origin_host={origin_host}, applying origin.set_url", error=f"Host mismatch for dashboard {dashboard_id}") try: origin.set_url(aligned_url) except Exception as e: - logger.warning( - "[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s", - dashboard_id, e, - ) + log.explore(f"Failed to set origin URL for dashboard {dashboard_id}: {e}", error=str(e)) return None try: session = SessionLocal() @@ -144,10 +141,7 @@ class GitServiceUrlMixin: finally: session.close() except Exception as e: - logger.warning( - "[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s", - dashboard_id, e, - ) + log.explore(f"Failed to persist aligned remote_url for dashboard {dashboard_id}: {e}", error=str(e)) return aligned_url # [/DEF:_align_origin_host_with_config:Function] diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index d1bdcb4a..3d0da734 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -15,7 +15,10 @@ from datetime import datetime, timezone from ..core.superset_client import SupersetClient from ..core.task_manager.models import Task from ..services.git_service import GitService +from ..core.cot_logger import MarkerLogger from ..core.logger import logger, belief_scope + +log = MarkerLogger("ResourceService") # [/SECTION] # [DEF:ResourceService:Class] @@ -33,7 +36,7 @@ class ResourceService: def __init__(self): with belief_scope("ResourceService.__init__"): self.git_service = GitService() - logger.info("[ResourceService][Action] Initialized ResourceService") + log.reason("Initialized ResourceService") # [/DEF:ResourceService_init:Function] # [DEF:get_dashboards_with_status:Function] @@ -82,7 +85,7 @@ class ResourceService: result.append(dashboard_dict) - logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status") + log.reflect(f"Fetched {len(result)} dashboards with status") return result # [/DEF:get_dashboards_with_status:Function] @@ -139,13 +142,7 @@ class ResourceService: result.append(dashboard_dict) total_pages = (total + page_size - 1) // page_size if total > 0 else 1 - logger.info( - "[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)", - page, - total_pages, - len(result), - total, - ) + log.reflect(f"Fetched dashboards page {page}/{total_pages} ({len(result)} items, total={total})") return { "dashboards": result, "total": total, @@ -324,7 +321,7 @@ class ResourceService: result.append(dataset_dict) - logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status") + log.reflect(f"Fetched {len(result)} datasets with status") return result # [/DEF:get_datasets_with_status:Function] @@ -413,7 +410,7 @@ class ResourceService: 'has_changes_for_commit': has_changes_for_commit } except Exception: - logger.warning(f"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}") + log.explore(f"Failed to get git status for dashboard {dashboard_id}", error="Git status check failed") return { 'branch': None, 'sync_status': 'ERROR', diff --git a/backend/tests/test_logger.py b/backend/tests/test_logger.py index be85a5b3..e894870f 100644 --- a/backend/tests/test_logger.py +++ b/backend/tests/test_logger.py @@ -1,29 +1,53 @@ # [DEF:TestLogger:Module] # @COMPLEXITY: 3 -# @SEMANTICS: logging, tests, belief_state -# @PURPOSE: Unit tests for the custom logger formatters and configuration context manager. +# @SEMANTICS: logging, tests, belief_state, cot, json +# @PURPOSE: Unit tests for the custom logger CoT JSON formatters and configuration context manager. # @LAYER: Logging (Tests) # @RELATION: VERIFIES -> src/core/logger.py # @INVARIANT: All required log statements must correctly check the threshold. import pytest +import logging + from src.core.logger import ( belief_scope, logger, configure_logger, get_task_log_level, - should_log_task_level + should_log_task_level, + CotJsonFormatter, ) from src.core.config_models import LoggingConfig -# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function] +@pytest.fixture(autouse=True) +def reset_logger_state(): + """Reset logger state before each test to avoid cross-test contamination.""" + config = LoggingConfig( + level="INFO", + task_log_level="INFO", + enable_belief_state=True + ) + configure_logger(config) + # Also reset the logger level for caplog to work correctly + logging.getLogger("superset_tools_app").setLevel(logging.DEBUG) + yield + # Reset after test too + config = LoggingConfig( + level="INFO", + task_log_level="INFO", + enable_belief_state=True + ) + configure_logger(config) + + +# [DEF:test_belief_scope_logs_reason_reflect_at_debug:Function] # @RELATION: BINDS_TO -> TestLogger -# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level. +# @PURPOSE: Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level. # @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level. -def test_belief_scope_logs_entry_action_exit_at_debug(caplog): - """Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.""" +# @POST: Logs are verified to contain REASON (entry) and REFLECT (coherence) markers. +def test_belief_scope_logs_reason_reflect_at_debug(caplog): + """Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -31,32 +55,43 @@ def test_belief_scope_logs_entry_action_exit_at_debug(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("TestFunction"): logger.info("Doing something important") - # Check that the logs contain the expected patterns - log_messages = [record.message for record in caplog.records] + # Check that the records contain the expected markers + reason_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction' + ] + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction' + ] + info_records = [ + r for r in caplog.records + if r.levelname == 'INFO' and 'Doing something important' in r.getMessage() + ] + + assert len(reason_records) >= 1, "REASON marker not found for TestFunction" + assert len(reflect_records) >= 1, "REFLECT marker not found for TestFunction" + assert len(info_records) >= 1, "INFO log 'Doing something important' not found" - assert any("[TestFunction][Entry]" in msg for msg in log_messages), "Entry log not found" - assert any("[TestFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found" - assert any("[TestFunction][Exit]" in msg for msg in log_messages), "Exit log not found" - # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) -# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function] +# [/DEF:test_belief_scope_logs_reason_reflect_at_debug:Function] # [DEF:test_belief_scope_error_handling:Function] # @RELATION: BINDS_TO -> TestLogger -# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception. +# @PURPOSE: Test that belief_scope logs EXPLORE marker on exception. # @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST: Logs are verified to contain Coherence:Failed tag. +# @POST: Logs are verified to contain EXPLORE marker with error context. def test_belief_scope_error_handling(caplog): - """Test that belief_scope logs Coherence:Failed on exception.""" + """Test that belief_scope logs EXPLORE marker on exception.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -64,19 +99,24 @@ def test_belief_scope_error_handling(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with pytest.raises(ValueError): with belief_scope("FailingFunction"): raise ValueError("Something went wrong") - log_messages = [record.message for record in caplog.records] + # Check that an EXPLORE marker was emitted + explore_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'EXPLORE' + and getattr(r, 'src', None) == 'FailingFunction' + ] + + assert len(explore_records) >= 1, f"EXPLORE marker not found for FailingFunction. Logs: {[(r.levelname, getattr(r, 'marker', ''), r.getMessage()) for r in caplog.records]}" + assert 'Something went wrong' in explore_records[0].getMessage() or \ + 'Something went wrong' in str(getattr(explore_records[0], 'error', '')) - assert any("[FailingFunction][Entry]" in msg for msg in log_messages), f"Entry log not found. Logs: {log_messages}" - assert any("[FailingFunction][COHERENCE:FAILED]" in msg for msg in log_messages), f"Failed coherence log not found. Logs: {log_messages}" - # Exit should not be logged on failure - # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) @@ -85,11 +125,11 @@ def test_belief_scope_error_handling(caplog): # [DEF:test_belief_scope_success_coherence:Function] # @RELATION: BINDS_TO -> TestLogger -# @PURPOSE: Test that belief_scope logs Coherence:OK on success. +# @PURPOSE: Test that belief_scope logs REFLECT marker on success. # @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST: Logs are verified to contain Coherence:OK tag. +# @POST: Logs are verified to contain REFLECT marker. def test_belief_scope_success_coherence(caplog): - """Test that belief_scope logs Coherence:OK on success.""" + """Test that belief_scope logs REFLECT marker on success.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -97,43 +137,58 @@ def test_belief_scope_success_coherence(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("SuccessFunction"): pass - log_messages = [record.message for record in caplog.records] + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' + and getattr(r, 'src', None) == 'SuccessFunction' + ] + + assert len(reflect_records) >= 1, f"REFLECT marker not found for SuccessFunction. Logs: {[(r.levelname, getattr(r, 'marker', ''), r.getMessage()) for r in caplog.records]}" + assert 'Coherence OK' in reflect_records[0].getMessage() or \ + getattr(reflect_records[0], 'intent', '') == 'Coherence OK' + - assert any("[SuccessFunction][COHERENCE:OK]" in msg for msg in log_messages), f"Success coherence log not found. Logs: {log_messages}" - - # Reset to INFO - config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) - configure_logger(config) # [/DEF:test_belief_scope_success_coherence:Function] -# [DEF:test_belief_scope_not_visible_at_info:Function] +# [DEF:test_belief_scope_reason_not_visible_at_info:Function] # @RELATION: BINDS_TO -> TestLogger -# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level. +# @PURPOSE: Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level. # @PRE: belief_scope is available. caplog fixture is used. -# @POST: Entry/Exit/Coherence logs are not captured at INFO level. -def test_belief_scope_not_visible_at_info(caplog): - """Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.""" +# @POST: REASON/REFLECT markers are not captured at INFO level. +def test_belief_scope_reason_not_visible_at_info(caplog): + """Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.""" caplog.set_level("INFO") with belief_scope("InfoLevelFunction"): logger.info("Doing something important") - log_messages = [record.message for record in caplog.records] + # The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO + reason_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction' + ] + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction' + ] - # Action log should be visible - assert any("[InfoLevelFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found" - # Entry/Exit/Coherence should NOT be visible at INFO level - assert not any("[InfoLevelFunction][Entry]" in msg for msg in log_messages), "Entry log should not be visible at INFO" - assert not any("[InfoLevelFunction][Exit]" in msg for msg in log_messages), "Exit log should not be visible at INFO" - assert not any("[InfoLevelFunction][Coherence:OK]" in msg for msg in log_messages), "Coherence log should not be visible at INFO" -# [/DEF:test_belief_scope_not_visible_at_info:Function] + assert len(reason_records) == 0, "REASON marker should not be visible at INFO" + assert len(reflect_records) == 0, "REFLECT marker should not be visible at INFO" + + # But the INFO-level message should be visible + info_records = [ + r for r in caplog.records + if r.levelname == 'INFO' and 'Doing something important' in r.getMessage() + ] + assert len(info_records) >= 1, "INFO log 'Doing something important' should be visible" +# [/DEF:test_belief_scope_reason_not_visible_at_info:Function] # [DEF:test_task_log_level_default:Function] @@ -176,10 +231,10 @@ def test_configure_logger_task_log_level(): enable_belief_state=True ) configure_logger(config) - + assert get_task_log_level() == "DEBUG", "task_log_level should be DEBUG" assert should_log_task_level("DEBUG") is True, "DEBUG should be logged at DEBUG threshold" - + # Reset to INFO config = LoggingConfig( level="INFO", @@ -193,11 +248,11 @@ def test_configure_logger_task_log_level(): # [DEF:test_enable_belief_state_flag:Function] # @RELATION: BINDS_TO -> TestLogger -# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging. +# @PURPOSE: Test that enable_belief_state flag controls belief_scope entry logging. # @PRE: LoggingConfig is available. caplog fixture is used. -# @POST: belief_scope logs are controlled by the flag. +# @POST: REASON entry marker suppressed when disabled; REFLECT coherence still logged. def test_enable_belief_state_flag(caplog): - """Test that enable_belief_state flag controls belief_scope logging.""" + """Test that enable_belief_state flag controls belief_scope REASON entry logging.""" # Disable belief state config = LoggingConfig( level="DEBUG", @@ -205,20 +260,26 @@ def test_enable_belief_state_flag(caplog): enable_belief_state=False ) configure_logger(config) - + caplog.set_level("DEBUG") - + with belief_scope("DisabledFunction"): logger.info("Doing something") - - log_messages = [record.message for record in caplog.records] - - # Entry and Exit should NOT be logged when disabled - assert not any("[DisabledFunction][Entry]" in msg for msg in log_messages), "Entry should not be logged when disabled" - assert not any("[DisabledFunction][Exit]" in msg for msg in log_messages), "Exit should not be logged when disabled" - # Coherence:OK should still be logged (internal tracking) - assert any("[DisabledFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence should still be logged" - + + # REASON entry marker should NOT be logged when disabled + reason_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction' + ] + assert len(reason_records) == 0, "REASON entry should not be logged when disabled" + + # REFLECT coherence marker should still be logged (internal tracking) + reflect_records = [ + r for r in caplog.records + if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction' + ] + assert len(reflect_records) >= 1, "REFLECT coherence should still be logged" + # Re-enable for other tests config = LoggingConfig( level="DEBUG", @@ -227,4 +288,73 @@ def test_enable_belief_state_flag(caplog): ) configure_logger(config) # [/DEF:test_enable_belief_state_flag:Function] -# [/DEF:TestLogger:Module] \ No newline at end of file + +# [DEF:test_cot_json_formatter_output:Function] +# @RELATION: BINDS_TO -> TestLogger +# @PURPOSE: Test that CotJsonFormatter produces valid JSON with expected fields. +def test_cot_json_formatter_output(): + """Test that CotJsonFormatter produces valid JSON with expected fields.""" + import json + from datetime import datetime, timezone + + formatter = CotJsonFormatter() + + # Create a LogRecord with structured extra data + record = logging.LogRecord( + name="test.module", + level=logging.INFO, + pathname="/fake/path.py", + lineno=42, + msg="Test action", + args=(), + exc_info=None, + ) + record.marker = "REASON" + record.intent = "Test action" + record.src = "TestModule" + record.payload = {"key": "value"} + + output = formatter.format(record) + parsed = json.loads(output) + + assert parsed["level"] == "INFO" + assert parsed["marker"] == "REASON" + assert parsed["intent"] == "Test action" + assert parsed["src"] == "TestModule" + assert parsed["payload"] == {"key": "value"} + assert "ts" in parsed + assert "trace_id" in parsed +# [/DEF:test_cot_json_formatter_output:Function] + +# [DEF:test_cot_json_formatter_plain_message:Function] +# @RELATION: BINDS_TO -> TestLogger +# @PURPOSE: Test that CotJsonFormatter wraps plain messages (no extra) with default marker. +def test_cot_json_formatter_plain_message(): + """Test that CotJsonFormatter wraps plain messages with default marker.""" + import json + + formatter = CotJsonFormatter() + + # Create a LogRecord WITHOUT extra data (plain message) + record = logging.LogRecord( + name="test.module", + level=logging.INFO, + pathname="/fake/path.py", + lineno=42, + msg="Plain info message", + args=(), + exc_info=None, + ) + + output = formatter.format(record) + parsed = json.loads(output) + + assert parsed["level"] == "INFO" + assert parsed["marker"] == "REASON" # default for plain messages + assert parsed["intent"] == "Plain info message" + assert parsed["src"] == "test.module" + assert "ts" in parsed + assert "trace_id" in parsed +# [/DEF:test_cot_json_formatter_plain_message:Function] + +# [/DEF:TestLogger:Module]