From c6189876b3cdf9a95b5e5eb0266eebce62534a7a Mon Sep 17 00:00:00 2001 From: busya Date: Thu, 14 May 2026 11:20:17 +0300 Subject: [PATCH] chore(lint): apply ruff --fix (4443 auto-fixes) Auto-fixed categories: - F401: unused imports removed - I001: import blocks sorted - W293: trailing whitespace stripped - UP035: deprecated typing imports replaced - SIM: simplify suggestions applied - ARG: unused args prefixed with underscore - T201: print statements removed - F841: unused variables removed - RUF059: unpacked variables prefixed Remaining ~1500 unfixable errors (C901, B904, N806, E402) require manual work. Backend smoke tests: 13/13 passed. --- backend/delete_running_tasks.py | 10 +- backend/src/api/auth.py | 14 +- backend/src/api/routes/__init__.py | 26 +- backend/src/api/routes/__tests__/conftest.py | 2 +- .../routes/__tests__/test_assistant_api.py | 18 +- .../routes/__tests__/test_assistant_authz.py | 9 +- .../__tests__/test_clean_release_api.py | 20 +- .../test_clean_release_legacy_compat.py | 32 +-- .../test_clean_release_source_policy.py | 14 +- .../__tests__/test_clean_release_v2_api.py | 15 +- .../test_clean_release_v2_release_api.py | 11 +- .../__tests__/test_connections_routes.py | 6 +- .../api/routes/__tests__/test_dashboards.py | 29 +- .../__tests__/test_dataset_review_api.py | 56 ++-- .../src/api/routes/__tests__/test_datasets.py | 17 +- .../src/api/routes/__tests__/test_git_api.py | 12 +- .../routes/__tests__/test_git_status_route.py | 10 +- .../routes/__tests__/test_migration_routes.py | 16 +- .../api/routes/__tests__/test_profile_api.py | 11 +- .../api/routes/__tests__/test_reports_api.py | 10 +- .../__tests__/test_reports_detail_api.py | 6 +- .../test_reports_openapi_conformance.py | 6 +- .../api/routes/__tests__/test_tasks_logs.py | 10 +- backend/src/api/routes/admin.py | 35 +-- backend/src/api/routes/assistant/__init__.py | 142 ++++----- .../src/api/routes/assistant/_admin_routes.py | 27 +- .../api/routes/assistant/_command_parser.py | 27 +- .../api/routes/assistant/_dataset_review.py | 31 +- .../assistant/_dataset_review_dispatch.py | 13 +- backend/src/api/routes/assistant/_dispatch.py | 39 +-- backend/src/api/routes/assistant/_history.py | 36 +-- .../src/api/routes/assistant/_llm_planner.py | 31 +- .../routes/assistant/_llm_planner_intent.py | 28 +- .../src/api/routes/assistant/_resolvers.py | 45 ++- backend/src/api/routes/assistant/_routes.py | 62 ++-- backend/src/api/routes/assistant/_schemas.py | 32 +-- backend/src/api/routes/clean_release.py | 51 ++-- backend/src/api/routes/clean_release_v2.py | 46 +-- backend/src/api/routes/connections.py | 35 +-- backend/src/api/routes/dashboards/__init__.py | 6 +- .../api/routes/dashboards/_action_routes.py | 11 +- .../api/routes/dashboards/_detail_routes.py | 29 +- backend/src/api/routes/dashboards/_helpers.py | 17 +- .../api/routes/dashboards/_listing_routes.py | 51 ++-- .../src/api/routes/dashboards/_projection.py | 38 +-- backend/src/api/routes/dashboards/_schemas.py | 97 ++++--- backend/src/api/routes/dataset_review.py | 36 +-- .../dataset_review_pkg/_dependencies.py | 76 ++--- .../api/routes/dataset_review_pkg/_routes.py | 75 +++-- backend/src/api/routes/datasets.py | 137 ++++----- backend/src/api/routes/environments.py | 25 +- backend/src/api/routes/git/__init__.py | 38 +-- backend/src/api/routes/git/_config_routes.py | 15 +- .../src/api/routes/git/_environment_routes.py | 6 +- backend/src/api/routes/git/_gitea_routes.py | 16 +- backend/src/api/routes/git/_helpers.py | 24 +- backend/src/api/routes/git/_merge_routes.py | 22 +- .../api/routes/git/_repo_lifecycle_routes.py | 25 +- .../api/routes/git/_repo_operations_routes.py | 38 ++- backend/src/api/routes/git/_repo_routes.py | 30 +- backend/src/api/routes/git_schemas.py | 81 +++--- backend/src/api/routes/health.py | 11 +- backend/src/api/routes/llm.py | 23 +- backend/src/api/routes/mappings.py | 21 +- backend/src/api/routes/migration.py | 36 +-- backend/src/api/routes/plugins.py | 6 +- backend/src/api/routes/profile.py | 7 +- backend/src/api/routes/reports.py | 19 +- backend/src/api/routes/settings.py | 51 ++-- backend/src/api/routes/storage.py | 19 +- backend/src/api/routes/tasks.py | 41 +-- backend/src/api/routes/translate/__init__.py | 18 +- .../routes/translate/_correction_routes.py | 12 +- .../routes/translate/_dictionary_routes.py | 13 +- backend/src/api/routes/translate/_helpers.py | 6 +- .../src/api/routes/translate/_job_routes.py | 32 +-- .../api/routes/translate/_metrics_routes.py | 12 +- .../api/routes/translate/_preview_routes.py | 25 +- .../api/routes/translate/_run_list_routes.py | 38 +-- .../src/api/routes/translate/_run_routes.py | 31 +- .../api/routes/translate/_schedule_routes.py | 15 +- backend/src/app.py | 66 +++-- .../__tests__/test_config_manager_compat.py | 5 +- .../src/core/__tests__/test_native_filters.py | 11 +- .../test_superset_preview_pipeline.py | 6 +- .../__tests__/test_superset_profile_lookup.py | 16 +- .../__tests__/test_throttled_scheduler.py | 8 +- backend/src/core/async_superset_client.py | 48 ++-- backend/src/core/auth/__tests__/test_auth.py | 17 +- backend/src/core/auth/config.py | 3 +- backend/src/core/auth/jwt.py | 7 +- backend/src/core/auth/logger.py | 6 +- backend/src/core/auth/oauth.py | 3 +- backend/src/core/auth/repository.py | 27 +- backend/src/core/auth/security.py | 1 + backend/src/core/config_manager.py | 24 +- backend/src/core/config_models.py | 15 +- backend/src/core/cot_logger.py | 30 +- backend/src/core/database.py | 28 +- backend/src/core/encryption_key.py | 2 +- backend/src/core/logger.py | 16 +- .../src/core/logger/__tests__/test_logger.py | 62 ++-- backend/src/core/mapping_service.py | 27 +- backend/src/core/middleware/trace.py | 1 + backend/src/core/migration/__init__.py | 4 +- backend/src/core/migration/archive_parser.py | 24 +- .../core/migration/dry_run_orchestrator.py | 85 +++--- backend/src/core/migration/risk_assessor.py | 24 +- backend/src/core/migration_engine.py | 34 ++- backend/src/core/plugin_base.py | 20 +- backend/src/core/plugin_loader.py | 24 +- backend/src/core/scheduler.py | 15 +- backend/src/core/superset_client/__init__.py | 10 +- backend/src/core/superset_client/_base.py | 31 +- backend/src/core/superset_client/_charts.py | 16 +- .../core/superset_client/_dashboards_crud.py | 36 +-- .../superset_client/_dashboards_filters.py | 31 +- .../core/superset_client/_dashboards_list.py | 31 +- .../src/core/superset_client/_databases.py | 19 +- backend/src/core/superset_client/_datasets.py | 23 +- .../core/superset_client/_datasets_preview.py | 36 +-- .../_datasets_preview_filters.py | 32 ++- .../core/superset_client/_user_projection.py | 20 +- backend/src/core/superset_profile_lookup.py | 27 +- backend/src/core/task_manager/__init__.py | 4 +- .../task_manager/__tests__/test_context.py | 5 +- .../__tests__/test_task_logger.py | 14 +- backend/src/core/task_manager/cleanup.py | 28 +- backend/src/core/task_manager/context.py | 18 +- backend/src/core/task_manager/manager.py | 59 ++-- backend/src/core/task_manager/models.py | 37 +-- backend/src/core/task_manager/persistence.py | 35 +-- backend/src/core/task_manager/task_logger.py | 29 +- backend/src/core/utils/async_network.py | 25 +- backend/src/core/utils/dataset_mapper.py | 31 +- backend/src/core/utils/fileio.py | 204 ++++++------- backend/src/core/utils/matching.py | 15 +- backend/src/core/utils/network.py | 102 +++---- .../utils/superset_compilation_adapter.py | 28 +- .../superset_context_extractor/__init__.py | 6 +- .../utils/superset_context_extractor/_base.py | 51 ++-- .../superset_context_extractor/_filters.py | 13 +- .../superset_context_extractor/_parsing.py | 19 +- .../utils/superset_context_extractor/_pii.py | 15 +- .../superset_context_extractor/_recovery.py | 23 +- .../superset_context_extractor/_templates.py | 29 +- backend/src/core/ws_log_handler.py | 6 +- backend/src/dependencies.py | 57 ++-- .../models/__tests__/test_clean_release.py | 31 +- backend/src/models/__tests__/test_models.py | 5 +- .../models/__tests__/test_report_models.py | 16 +- backend/src/models/assistant.py | 2 +- backend/src/models/auth.py | 4 +- backend/src/models/clean_release.py | 84 +++--- backend/src/models/config.py | 2 +- backend/src/models/connection.py | 11 +- backend/src/models/dashboard.py | 7 +- backend/src/models/dataset_review.py | 92 +++--- .../src/models/dataset_review_pkg/__init__.py | 178 ++++++------ .../_clarification_models.py | 12 +- .../dataset_review_pkg/_execution_models.py | 16 +- .../dataset_review_pkg/_filter_models.py | 16 +- .../dataset_review_pkg/_finding_models.py | 8 +- .../dataset_review_pkg/_mapping_models.py | 14 +- .../dataset_review_pkg/_profile_models.py | 12 +- .../dataset_review_pkg/_semantic_models.py | 24 +- .../dataset_review_pkg/_session_models.py | 12 +- backend/src/models/filter_state.py | 79 ++--- backend/src/models/git.py | 7 +- backend/src/models/llm.py | 13 +- backend/src/models/mapping.py | 8 +- backend/src/models/profile.py | 4 +- backend/src/models/report.py | 34 +-- backend/src/models/storage.py | 7 +- backend/src/models/task.py | 11 +- backend/src/models/translate.py | 45 ++- backend/src/plugins/backup.py | 56 ++-- backend/src/plugins/debug.py | 38 +-- backend/src/plugins/git/llm_extension.py | 16 +- backend/src/plugins/git_plugin.py | 79 ++--- backend/src/plugins/llm_analysis/models.py | 19 +- backend/src/plugins/llm_analysis/plugin.py | 94 +++--- backend/src/plugins/llm_analysis/scheduler.py | 14 +- backend/src/plugins/llm_analysis/service.py | 152 +++++----- backend/src/plugins/mapper.py | 32 ++- backend/src/plugins/migration.py | 153 +++++----- backend/src/plugins/search.py | 28 +- backend/src/plugins/storage/plugin.py | 56 ++-- .../test_clickhouse_insert_integration.py | 31 +- .../translate/__tests__/test_dictionary.py | 11 +- .../translate/__tests__/test_orchestrator.py | 56 ++-- .../translate/__tests__/test_preview.py | 16 +- .../translate/__tests__/test_sql_generator.py | 27 +- backend/src/plugins/translate/_utils.py | 1 - backend/src/plugins/translate/dictionary.py | 66 ++--- backend/src/plugins/translate/events.py | 35 +-- backend/src/plugins/translate/executor.py | 56 ++-- backend/src/plugins/translate/metrics.py | 21 +- backend/src/plugins/translate/orchestrator.py | 49 ++-- backend/src/plugins/translate/plugin.py | 7 +- backend/src/plugins/translate/preview.py | 56 ++-- backend/src/plugins/translate/scheduler.py | 37 ++- backend/src/plugins/translate/service.py | 39 +-- .../src/plugins/translate/sql_generator.py | 57 ++-- .../plugins/translate/superset_executor.py | 33 ++- .../test_settings_and_health_schemas.py | 5 +- backend/src/schemas/auth.py | 40 +-- backend/src/schemas/dataset_review.py | 26 +- .../schemas/dataset_review_pkg/_composites.py | 88 +++--- .../src/schemas/dataset_review_pkg/_dtos.py | 98 +++---- backend/src/schemas/health.py | 16 +- backend/src/schemas/profile.py | 69 +++-- backend/src/schemas/settings.py | 40 +-- backend/src/schemas/translate.py | 270 +++++++++--------- backend/src/scripts/clean_release_cli.py | 20 +- backend/src/scripts/clean_release_tui.py | 46 +-- backend/src/scripts/create_admin.py | 8 +- backend/src/scripts/init_auth_db.py | 4 +- .../src/scripts/migrate_sqlite_to_postgres.py | 16 +- backend/src/scripts/seed_permissions.py | 6 +- .../src/scripts/seed_superset_load_test.py | 25 +- .../test_dataset_dashboard_relations.py | 52 ++-- .../__tests__/test_encryption_manager.py | 11 +- .../services/__tests__/test_health_service.py | 6 +- .../__tests__/test_llm_plugin_persistence.py | 1 + .../__tests__/test_llm_prompt_templates.py | 4 +- .../services/__tests__/test_llm_provider.py | 8 +- .../__tests__/test_rbac_permission_catalog.py | 3 +- .../__tests__/test_resource_service.py | 9 +- backend/src/services/auth_service.py | 15 +- .../__tests__/test_audit_service.py | 3 +- .../__tests__/test_policy_engine.py | 8 +- .../__tests__/test_preparation_service.py | 18 +- .../__tests__/test_report_builder.py | 8 +- .../__tests__/test_source_isolation.py | 4 +- .../clean_release/__tests__/test_stages.py | 3 +- .../clean_release/approval_service.py | 15 +- .../clean_release/artifact_catalog_loader.py | 5 +- .../services/clean_release/audit_service.py | 20 +- .../clean_release/candidate_service.py | 13 +- .../compliance_execution_service.py | 71 +++-- .../clean_release/compliance_orchestrator.py | 33 +-- .../clean_release/demo_data_service.py | 2 +- backend/src/services/clean_release/dto.py | 55 ++-- backend/src/services/clean_release/enums.py | 3 +- .../src/services/clean_release/exceptions.py | 2 +- backend/src/services/clean_release/facade.py | 34 +-- .../clean_release/manifest_builder.py | 13 +- .../clean_release/manifest_service.py | 4 +- backend/src/services/clean_release/mappers.py | 21 +- .../services/clean_release/policy_engine.py | 20 +- .../policy_resolution_service.py | 8 +- .../clean_release/preparation_service.py | 18 +- .../clean_release/publication_service.py | 15 +- .../services/clean_release/report_builder.py | 11 +- .../clean_release/repositories/__init__.py | 24 +- .../repositories/approval_repository.py | 14 +- .../repositories/artifact_repository.py | 14 +- .../repositories/audit_repository.py | 17 +- .../repositories/candidate_repository.py | 14 +- .../repositories/compliance_repository.py | 18 +- .../repositories/manifest_repository.py | 13 +- .../repositories/policy_repository.py | 11 +- .../repositories/publication_repository.py | 14 +- .../repositories/report_repository.py | 14 +- .../src/services/clean_release/repository.py | 80 +++--- .../clean_release/source_isolation.py | 6 +- .../services/clean_release/stages/__init__.py | 19 +- .../src/services/clean_release/stages/base.py | 12 +- .../src/services/dataset_review/__init__.py | 2 +- .../dataset_review/clarification_engine.py | 35 ++- .../clarification_pkg/_helpers.py | 18 +- .../services/dataset_review/event_logger.py | 13 +- .../services/dataset_review/orchestrator.py | 78 +++-- .../orchestrator_pkg/_commands.py | 15 +- .../orchestrator_pkg/_helpers.py | 45 ++- .../__tests__/test_session_repository.py | 20 +- .../repositories/repository_pkg/_mutations.py | 25 +- .../repositories/session_repository.py | 33 +-- .../dataset_review/semantic_resolver.py | 35 +-- backend/src/services/git/__init__.py | 8 +- backend/src/services/git/_base.py | 23 +- backend/src/services/git/_branch.py | 14 +- backend/src/services/git/_gitea.py | 28 +- backend/src/services/git/_merge.py | 26 +- backend/src/services/git/_remote_providers.py | 34 +-- backend/src/services/git/_status.py | 7 +- backend/src/services/git/_sync.py | 12 +- backend/src/services/git/_url.py | 25 +- backend/src/services/health_service.py | 40 +-- backend/src/services/llm_prompt_templates.py | 17 +- backend/src/services/llm_provider.py | 22 +- backend/src/services/mapping_service.py | 4 +- .../__tests__/test_notification_service.py | 32 +-- .../src/services/notifications/providers.py | 23 +- backend/src/services/notifications/service.py | 23 +- backend/src/services/profile_service.py | 62 ++-- .../src/services/rbac_permission_catalog.py | 23 +- .../reports/__tests__/test_report_service.py | 6 +- .../reports/__tests__/test_type_profiles.py | 6 +- backend/src/services/reports/normalizer.py | 10 +- .../src/services/reports/report_service.py | 20 +- backend/src/services/reports/type_profiles.py | 10 +- backend/src/services/resource_service.py | 118 ++++---- backend/test_pat_api.py | 7 +- backend/tests/core/test_defensive_guards.py | 10 +- .../tests/core/test_git_service_gitea_pr.py | 2 +- backend/tests/core/test_mapping_service.py | 12 +- backend/tests/core/test_migration_engine.py | 31 +- .../tests/scripts/test_clean_release_cli.py | 16 +- .../tests/scripts/test_clean_release_tui.py | 33 ++- .../clean_release/test_approval_service.py | 12 +- .../test_candidate_manifest_services.py | 13 +- .../test_compliance_execution_service.py | 6 +- .../test_compliance_task_integration.py | 10 +- .../clean_release/test_demo_mode_isolation.py | 6 +- .../clean_release/test_publication_service.py | 8 +- .../test_report_audit_immutability.py | 14 +- .../dataset_review/test_superset_matrix.py | 6 +- backend/tests/test_auth.py | 11 +- backend/tests/test_core_scheduler.py | 4 +- backend/tests/test_dashboards_api.py | 22 +- backend/tests/test_log_persistence.py | 5 +- backend/tests/test_logger.py | 15 +- backend/tests/test_models.py | 1 + backend/tests/test_resource_hubs.py | 6 +- backend/tests/test_smoke_app.py | 6 +- backend/tests/test_smoke_plugins.py | 25 +- backend/tests/test_task_manager.py | 6 +- backend/tests/test_task_persistence.py | 7 +- backend/tests/test_translate_corrections.py | 10 +- .../tests/test_translate_executor_filter.py | 2 - backend/tests/test_translate_history.py | 21 +- backend/tests/test_translate_jobs.py | 20 +- backend/tests/test_translate_scheduler.py | 16 +- .../test_translate_scheduler_execution.py | 8 +- .../tests/test_translate_scheduler_guard.py | 64 ++--- 337 files changed, 4677 insertions(+), 4515 deletions(-) diff --git a/backend/delete_running_tasks.py b/backend/delete_running_tasks.py index a9eefac0..8e6bdca2 100644 --- a/backend/delete_running_tasks.py +++ b/backend/delete_running_tasks.py @@ -7,9 +7,11 @@ # @RELATION: DEPENDS_ON ->[TaskRecord] from sqlalchemy.orm import Session + from src.core.database import TasksSessionLocal from src.models.task import TaskRecord + # [DEF:delete_running_tasks:Function] # @PURPOSE: Delete all tasks with RUNNING status from the database. # @PRE: Database is accessible and TaskRecord model is defined. @@ -20,19 +22,19 @@ def delete_running_tasks(): try: # Find all task records with RUNNING status running_tasks = session.query(TaskRecord).filter(TaskRecord.status == "RUNNING").all() - + if not running_tasks: print("No RUNNING tasks found.") return - + print(f"Found {len(running_tasks)} RUNNING tasks:") for task in running_tasks: print(f"- Task ID: {task.id}, Type: {task.type}") - + # Delete the found tasks session.query(TaskRecord).filter(TaskRecord.status == "RUNNING").delete(synchronize_session=False) session.commit() - + print(f"Successfully deleted {len(running_tasks)} RUNNING tasks.") except Exception as e: session.rollback() diff --git a/backend/src/api/auth.py b/backend/src/api/auth.py index 430cde6d..738c11eb 100755 --- a/backend/src/api/auth.py +++ b/backend/src/api/auth.py @@ -8,17 +8,19 @@ # @RELATION DEPENDS_ON -> [is_adfs_configured] # @INVARIANT: All auth endpoints must return consistent error codes. +import starlette.requests from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordRequestForm from sqlalchemy.orm import Session -from ..core.database import get_auth_db -from ..services.auth_service import AuthService -from ..schemas.auth import Token, User as UserSchema -from ..dependencies import get_current_user -from ..core.auth.oauth import oauth, is_adfs_configured + from ..core.auth.logger import log_security_event +from ..core.auth.oauth import is_adfs_configured, oauth +from ..core.database import get_auth_db from ..core.logger import belief_scope -import starlette.requests +from ..dependencies import get_current_user +from ..schemas.auth import Token +from ..schemas.auth import User as UserSchema +from ..services.auth_service import AuthService # #region router [C:1] [TYPE Variable] # @RELATION DEPENDS_ON -> [fastapi.APIRouter] diff --git a/backend/src/api/routes/__init__.py b/backend/src/api/routes/__init__.py index 4fa30563..726cf0cf 100755 --- a/backend/src/api/routes/__init__.py +++ b/backend/src/api/routes/__init__.py @@ -14,26 +14,26 @@ # @RELATION DEPENDS_ON -> [ReportsRouter] # @RELATION DEPENDS_ON -> [LlmRoutes] __all__ = [ - "plugins", - "tasks", - "settings", - "connections", - "environments", - "mappings", - "migration", - "git", - "storage", "admin", - "reports", "assistant", "clean_release", "clean_release_v2", - "profile", - "dataset_review", - "llm", + "connections", "dashboards", + "dataset_review", "datasets", + "environments", + "git", "health", + "llm", + "mappings", + "migration", + "plugins", + "profile", + "reports", + "settings", + "storage", + "tasks", "translate", ] # #endregion Route_Group_Contracts diff --git a/backend/src/api/routes/__tests__/conftest.py b/backend/src/api/routes/__tests__/conftest.py index 4f265ddd..6af8330c 100644 --- a/backend/src/api/routes/__tests__/conftest.py +++ b/backend/src/api/routes/__tests__/conftest.py @@ -28,4 +28,4 @@ class FakeQuery: return list(self._rows) def count(self): return len(self._rows) -# #endregion RoutesTestsConftest \ No newline at end of file +# #endregion RoutesTestsConftest diff --git a/backend/src/api/routes/__tests__/test_assistant_api.py b/backend/src/api/routes/__tests__/test_assistant_api.py index 51f6749c..a24d260a 100644 --- a/backend/src/api/routes/__tests__/test_assistant_api.py +++ b/backend/src/api/routes/__tests__/test_assistant_api.py @@ -1,4 +1,5 @@ import os + os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" # #region AssistantApiTests [TYPE Module] [C:3] [SEMANTICS tests, assistant, api] # @BRIEF Validate assistant API endpoint logic via direct async handler invocation. @@ -6,14 +7,10 @@ os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" # @INVARIANT: Every test clears assistant in-memory state before execution. import asyncio import uuid -from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Tuple +from datetime import datetime from unittest.mock import MagicMock -import pytest -from fastapi import HTTPException -from pydantic import BaseModel + from src.api.routes import assistant as assistant_routes -from src.schemas.auth import User from src.models.assistant import AssistantMessageRecord from src.models.dataset_review import ( ApprovalState, @@ -22,13 +19,16 @@ from src.models.dataset_review import ( ExecutionMapping, ImportedFilter, MappingMethod, - SemanticCandidate, - SemanticFieldEntry, ReadinessState, RecommendedAction, + SemanticCandidate, + SemanticFieldEntry, SessionPhase, SessionStatus, ) +from src.schemas.auth import User + + # #region _run_async [TYPE Function] # @RELATION BINDS_TO -> [AssistantApiTests] def _run_async(coro): @@ -437,4 +437,4 @@ def test_dataset_review_scoped_command_routes_field_semantics_update(): assert resp.intent["entities"]["description"] == "Approved semantic wording" assert resp.intent["entities"]["lock_field"] is True # #endregion test_dataset_review_scoped_command_routes_field_semantics_update -# #endregion AssistantApiTests \ No newline at end of file +# #endregion AssistantApiTests diff --git a/backend/src/api/routes/__tests__/test_assistant_authz.py b/backend/src/api/routes/__tests__/test_assistant_authz.py index fea3e324..9cb37aff 100644 --- a/backend/src/api/routes/__tests__/test_assistant_authz.py +++ b/backend/src/api/routes/__tests__/test_assistant_authz.py @@ -1,16 +1,19 @@ import os + os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" # #region TestAssistantAuthz [TYPE Module] [C:3] [SEMANTICS tests, assistant, authz, confirmation, rbac] # @BRIEF Verify assistant confirmation ownership, expiration, and deny behavior for restricted users. # @LAYER: UI (API Tests) # @RELATION DEPENDS_ON -> AssistantApi # @INVARIANT: Security-sensitive flows fail closed for unauthorized actors. -import os import asyncio +import os from datetime import datetime, timedelta from types import SimpleNamespace + import pytest from fastapi import HTTPException + # Force isolated sqlite databases for test module before dependencies import. os.environ.setdefault("DATABASE_URL", "sqlite:////tmp/ss_tools_assistant_authz.db") os.environ.setdefault( @@ -25,6 +28,8 @@ from src.models.assistant import ( AssistantConfirmationRecord, AssistantMessageRecord, ) + + # #region _run_async [TYPE Function] [C:1] # @RELATION BINDS_TO -> [TestAssistantAuthz] # @BRIEF Execute async endpoint handler in synchronous test context. @@ -279,4 +284,4 @@ def test_limited_user_cannot_launch_restricted_operation(): ) assert response.state == "denied" # #endregion test_limited_user_cannot_launch_restricted_operation -# #endregion TestAssistantAuthz \ No newline at end of file +# #endregion TestAssistantAuthz diff --git a/backend/src/api/routes/__tests__/test_clean_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_api.py index 2fac409a..4f72502d 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_api.py @@ -3,21 +3,25 @@ # @BRIEF Contract tests for clean release checks and reports endpoints. # @LAYER: Domain # @INVARIANT: API returns deterministic payload shapes for checks and reports. -from datetime import datetime, timezone +from datetime import UTC, datetime + from fastapi.testclient import TestClient + from src.app import app from src.dependencies import get_clean_release_repository from src.models.clean_release import ( + CheckFinalStatus, CleanProfilePolicy, + ComplianceReport, ProfileType, ReleaseCandidate, ReleaseCandidateStatus, ResourceSourceEntry, ResourceSourceRegistry, - ComplianceReport, - CheckFinalStatus, ) from src.services.clean_release.repository import CleanReleaseRepository + + # #region _repo_with_seed_data [TYPE Function] # @RELATION BINDS_TO -> TestCleanReleaseApi def _repo_with_seed_data() -> CleanReleaseRepository: @@ -27,7 +31,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: candidate_id="2026.03.03-rc1", version="2026.03.03", profile=ProfileType.ENTERPRISE_CLEAN, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), created_by="tester", source_snapshot_ref="git:abc123", status=ReleaseCandidateStatus.PREPARED, @@ -46,7 +50,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: enabled=True, ) ], - updated_at=datetime.now(timezone.utc), + updated_at=datetime.now(UTC), updated_by="tester", status="active", ) @@ -60,7 +64,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: required_system_categories=["system-init"], external_source_forbidden=True, internal_source_registry_ref="registry-internal-v1", - effective_from=datetime.now(timezone.utc), + effective_from=datetime.now(UTC), profile=ProfileType.ENTERPRISE_CLEAN, ) ) @@ -120,7 +124,7 @@ def test_get_report_success(): report_id="rep-1", check_run_id="run-1", candidate_id="2026.03.03-rc1", - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), final_status=CheckFinalStatus.COMPLIANT, operator_summary="all systems go", structured_payload_ref="manifest-1", @@ -163,4 +167,4 @@ def test_prepare_candidate_api_success(): finally: app.dependency_overrides.clear() # #endregion test_prepare_candidate_api_success -# #endregion TestCleanReleaseApi \ No newline at end of file +# #endregion TestCleanReleaseApi diff --git a/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py b/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py index 2f51c278..bdaf5523 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py +++ b/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py @@ -3,9 +3,12 @@ # @BRIEF Compatibility tests for legacy clean-release API paths retained during v2 migration. # @LAYER: Tests from __future__ import annotations + import os -from datetime import datetime, timezone +from datetime import UTC, datetime + from fastapi.testclient import TestClient + os.environ.setdefault("DATABASE_URL", "sqlite:///./test_clean_release_legacy_compat.db") os.environ.setdefault( "AUTH_DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db" @@ -22,6 +25,8 @@ from src.models.clean_release import ( ResourceSourceRegistry, ) from src.services.clean_release.repository import CleanReleaseRepository + + # #region _seed_legacy_repo [TYPE Function] # @RELATION BINDS_TO -> TestCleanReleaseLegacyCompat # @BRIEF Seed in-memory repository with minimum trusted data for legacy endpoint contracts. @@ -29,7 +34,7 @@ from src.services.clean_release.repository import CleanReleaseRepository # @POST: Candidate, policy, registry and manifest are available for legacy checks flow. def _seed_legacy_repo() -> CleanReleaseRepository: repo = CleanReleaseRepository() - now = datetime.now(timezone.utc) + now = datetime.now(UTC) repo.save_candidate( ReleaseCandidate( id="legacy-rc-001", @@ -56,10 +61,10 @@ def _seed_legacy_repo() -> CleanReleaseRepository: updated_by="compat-tester", status="ACTIVE", ) - setattr(registry, "immutable", True) - setattr(registry, "allowed_hosts", ["repo.intra.company.local"]) - setattr(registry, "allowed_schemes", ["https"]) - setattr(registry, "allowed_source_types", ["artifact-repo"]) + registry.immutable = True + registry.allowed_hosts = ["repo.intra.company.local"] + registry.allowed_schemes = ["https"] + registry.allowed_source_types = ["artifact-repo"] repo.save_registry(registry) policy = CleanProfilePolicy( policy_id="legacy-pol-1", @@ -71,17 +76,8 @@ def _seed_legacy_repo() -> CleanReleaseRepository: required_system_categories=["core"], effective_from=now, ) - setattr(policy, "immutable", True) - setattr( - policy, - "content_json", - { - "profile": "enterprise-clean", - "prohibited_artifact_categories": ["test-data"], - "required_system_categories": ["core"], - "external_source_forbidden": True, - }, - ) + policy.immutable = True + policy.content_json = {"profile": "enterprise-clean", "prohibited_artifact_categories": ["test-data"], "required_system_categories": ["core"], "external_source_forbidden": True} repo.save_policy(policy) repo.save_manifest( DistributionManifest( @@ -160,4 +156,4 @@ def test_legacy_checks_endpoints_still_available() -> None: finally: app.dependency_overrides.clear() # #endregion test_legacy_checks_endpoints_still_available -# #endregion TestCleanReleaseLegacyCompat \ No newline at end of file +# #endregion TestCleanReleaseLegacyCompat diff --git a/backend/src/api/routes/__tests__/test_clean_release_source_policy.py b/backend/src/api/routes/__tests__/test_clean_release_source_policy.py index 35fa7999..091d4166 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_source_policy.py +++ b/backend/src/api/routes/__tests__/test_clean_release_source_policy.py @@ -3,8 +3,10 @@ # @BRIEF Validate API behavior for source isolation violations in clean release preparation. # @LAYER: Domain # @INVARIANT: External endpoints must produce blocking violation entries. -from datetime import datetime, timezone +from datetime import UTC, datetime + from fastapi.testclient import TestClient + from src.app import app from src.dependencies import get_clean_release_repository from src.models.clean_release import ( @@ -16,6 +18,8 @@ from src.models.clean_release import ( ResourceSourceRegistry, ) from src.services.clean_release.repository import CleanReleaseRepository + + # #region _repo_with_seed_data [TYPE Function] # @RELATION BINDS_TO -> TestCleanReleaseSourcePolicy # @BRIEF Seed repository with candidate, registry, and active policy for source isolation test flow. @@ -26,7 +30,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: candidate_id="2026.03.03-rc1", version="2026.03.03", profile=ProfileType.ENTERPRISE_CLEAN, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), created_by="tester", source_snapshot_ref="git:abc123", status=ReleaseCandidateStatus.DRAFT, @@ -45,7 +49,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: enabled=True, ) ], - updated_at=datetime.now(timezone.utc), + updated_at=datetime.now(UTC), updated_by="tester", status="active", ) @@ -59,7 +63,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: required_system_categories=["system-init"], external_source_forbidden=True, internal_source_registry_ref="registry-internal-v1", - effective_from=datetime.now(timezone.utc), + effective_from=datetime.now(UTC), profile=ProfileType.ENTERPRISE_CLEAN, ) ) @@ -95,4 +99,4 @@ def test_prepare_candidate_blocks_external_source(): finally: app.dependency_overrides.clear() # #endregion test_prepare_candidate_blocks_external_source -# #endregion TestCleanReleaseSourcePolicy \ No newline at end of file +# #endregion TestCleanReleaseSourcePolicy diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py index b9930f64..a4f074fe 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py @@ -2,20 +2,11 @@ # @BRIEF API contract tests for redesigned clean release endpoints. # @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseV2Api] -from datetime import datetime, timezone -from types import SimpleNamespace -from uuid import uuid4 -import pytest from fastapi.testclient import TestClient + from src.app import app -from src.dependencies import get_clean_release_repository, get_config_manager -from src.models.clean_release import ( - CleanPolicySnapshot, - DistributionManifest, - ReleaseCandidate, - SourceRegistrySnapshot, -) from src.services.clean_release.enums import CandidateStatus + client = TestClient(app) # [REASON] Implementing API contract tests for candidate/artifact/manifest endpoints (T012). # #region test_candidate_registration_contract [TYPE Function] @@ -93,4 +84,4 @@ def test_manifest_build_contract(): assert "manifest_digest" in data assert data["candidate_id"] == candidate_id # #endregion test_manifest_build_contract -# #endregion CleanReleaseV2ApiTests \ No newline at end of file +# #endregion CleanReleaseV2ApiTests diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py index 061060c2..82308a89 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py @@ -3,14 +3,17 @@ # @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseV2Api] """Contract tests for redesigned approval/publication API endpoints.""" -from datetime import datetime, timezone +from datetime import UTC, datetime from uuid import uuid4 + from fastapi import FastAPI from fastapi.testclient import TestClient + from src.api.routes.clean_release_v2 import router as clean_release_v2_router from src.dependencies import get_clean_release_repository from src.models.clean_release import ComplianceReport, ReleaseCandidate from src.services.clean_release.enums import CandidateStatus, ComplianceDecision + test_app = FastAPI() test_app.include_router(clean_release_v2_router) client = TestClient(test_app) @@ -27,7 +30,7 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]: version="1.0.0", source_snapshot_ref="git:sha-api-release", created_by="api-test", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.CHECK_PASSED.value, ) ) @@ -42,7 +45,7 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]: "violations_count": 0, "blocking_violations_count": 0, }, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) ) @@ -100,4 +103,4 @@ def test_release_reject_contract() -> None: assert payload["status"] == "ok" assert payload["decision"] == "REJECTED" # #endregion test_release_reject_contract -# #endregion CleanReleaseV2ReleaseApiTests \ No newline at end of file +# #endregion CleanReleaseV2ReleaseApiTests diff --git a/backend/src/api/routes/__tests__/test_connections_routes.py b/backend/src/api/routes/__tests__/test_connections_routes.py index b6292ec3..547a468c 100644 --- a/backend/src/api/routes/__tests__/test_connections_routes.py +++ b/backend/src/api/routes/__tests__/test_connections_routes.py @@ -2,14 +2,16 @@ # @BRIEF Verifies connection routes bootstrap their table before CRUD access. # @LAYER: API # @RELATION DEPENDS_ON -> ConnectionsRouter +import asyncio import os import sys -import asyncio from pathlib import Path + import pytest from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool + # Force SQLite in-memory for database module imports. # @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv. os.environ["DATABASE_URL"] = "sqlite:///:memory:" @@ -64,4 +66,4 @@ def test_create_connection_bootstraps_missing_table(db_session): assert created.host == "warehouse.internal" assert "connection_configs" in inspector.get_table_names() # #endregion test_create_connection_bootstraps_missing_table -# #endregion ConnectionsRoutesTests \ No newline at end of file +# #endregion ConnectionsRoutesTests diff --git a/backend/src/api/routes/__tests__/test_dashboards.py b/backend/src/api/routes/__tests__/test_dashboards.py index 9321a2f9..6219164c 100644 --- a/backend/src/api/routes/__tests__/test_dashboards.py +++ b/backend/src/api/routes/__tests__/test_dashboards.py @@ -2,22 +2,25 @@ # @BRIEF Unit tests for dashboards API endpoints. # @LAYER: API # @RELATION DEPENDS_ON -> [DashboardsApi] +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock, patch + import pytest -from unittest.mock import MagicMock, patch, AsyncMock -from datetime import datetime, timezone from fastapi.testclient import TestClient -from src.app import app + from src.api.routes.dashboards import DashboardsResponse -from src.dependencies import ( - get_current_user, - has_permission, - get_config_manager, - get_task_manager, - get_resource_service, - get_mapping_service, -) +from src.app import app from src.core.database import get_db +from src.dependencies import ( + get_config_manager, + get_current_user, + get_mapping_service, + get_resource_service, + get_task_manager, + has_permission, +) from src.services.profile_service import ProfileService as DomainProfileService + # Global mock user for get_current_user dependency overrides mock_user = MagicMock() mock_user.id = "u-1" @@ -404,7 +407,7 @@ def test_get_database_mappings_env_not_found(mock_deps): # @BRIEF Validate dashboard task history returns only related backup and LLM tasks. # @TEST: GET /api/dashboards/{id}/tasks returns backup and llm tasks for dashboard def test_get_dashboard_tasks_history_filters_success(mock_deps): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) llm_task = MagicMock() llm_task.id = "task-llm-1" llm_task.plugin_id = "llm_dashboard_validation" @@ -871,4 +874,4 @@ def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(moc assert {item["id"] for item in payload["dashboards"]} == {701} assert payload["dashboards"][0]["title"] == "Featured Charts" # #endregion test_get_dashboards_profile_filter_matches_owner_object_payload_contract -# #endregion DashboardsApiTests \ No newline at end of file +# #endregion DashboardsApiTests diff --git a/backend/src/api/routes/__tests__/test_dataset_review_api.py b/backend/src/api/routes/__tests__/test_dataset_review_api.py index c6406fde..1df26d92 100644 --- a/backend/src/api/routes/__tests__/test_dataset_review_api.py +++ b/backend/src/api/routes/__tests__/test_dataset_review_api.py @@ -3,23 +3,22 @@ # @LAYER: API # @RELATION [BINDS_TO] ->[DatasetReviewApi] # @RELATION [BINDS_TO] ->[DatasetReviewOrchestrator] -from datetime import datetime, timezone -import json +from datetime import UTC, datetime from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch + import pytest from fastapi.testclient import TestClient -from src.app import app + from src.api.routes.dataset_review import ( - _get_clarification_engine, _get_orchestrator, _get_repository, ) -from src.core.config_models import Environment, GlobalSettings, AppConfig +from src.app import app +from src.core.config_models import AppConfig, Environment, GlobalSettings from src.core.utils.superset_context_extractor import SupersetContextExtractor from src.dependencies import get_config_manager, get_current_user, get_task_manager from src.models.dataset_review import ( - AnswerKind, ApprovalState, BusinessSummarySource, CandidateMatchType, @@ -31,12 +30,12 @@ from src.models.dataset_review import ( CompiledPreview, ConfidenceState, DatasetReviewSession, - LaunchStatus, ExecutionMapping, FieldKind, FieldProvenance, FindingArea, FindingSeverity, + LaunchStatus, MappingMethod, PreviewStatus, QuestionState, @@ -46,12 +45,13 @@ from src.models.dataset_review import ( SemanticCandidate, SemanticFieldEntry, SemanticSource, - SessionPhase, - SessionStatus, SemanticSourceStatus, SemanticSourceType, + SessionPhase, + SessionStatus, TrustLevel, ) +from src.services.dataset_review.event_logger import SessionEventLogger from src.services.dataset_review.orchestrator import ( DatasetReviewOrchestrator, LaunchDatasetResult, @@ -61,11 +61,11 @@ from src.services.dataset_review.orchestrator import ( from src.services.dataset_review.orchestrator_pkg._helpers import ( build_execution_snapshot, ) -from src.services.dataset_review.semantic_resolver import SemanticSourceResolver -from src.services.dataset_review.event_logger import SessionEventLogger from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionVersionConflictError, ) +from src.services.dataset_review.semantic_resolver import SemanticSourceResolver + client = TestClient(app) # #region _make_user [TYPE Function] # @RELATION BINDS_TO -> DatasetReviewApiTests @@ -101,7 +101,7 @@ def _make_config_manager(): # #region _make_session [TYPE Function] # @RELATION BINDS_TO -> DatasetReviewApiTests def _make_session(): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) return DatasetReviewSession( session_id="sess-1", user_id="user-1", @@ -124,7 +124,7 @@ def _make_session(): # #region _make_us2_session [TYPE Function] # @RELATION BINDS_TO -> DatasetReviewApiTests def _make_us2_session(): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) session = _make_session() session.readiness_state = ReadinessState.CLARIFICATION_NEEDED session.recommended_action = RecommendedAction.START_CLARIFICATION @@ -239,7 +239,7 @@ def _make_us3_session(): `imported_filter` and `template_variable` are bare MagicMocks without spec; ORM attribute access is unchecked. """ - now = datetime.now(timezone.utc) + now = datetime.now(UTC) session = _make_session() session.readiness_state = ReadinessState.MAPPING_REVIEW_NEEDED session.recommended_action = RecommendedAction.APPROVE_MAPPING @@ -695,7 +695,7 @@ def test_start_session_endpoint_returns_created_summary( def test_get_session_detail_export_and_lifecycle_endpoints( dataset_review_api_dependencies, ): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) session = MagicMock(spec=DatasetReviewSession) session.session_id = "sess-1" session.user_id = "user-1" @@ -957,8 +957,8 @@ def test_us3_mapping_patch_approval_preview_and_launch_endpoints( compiled_by="superset", error_code=None, error_details=None, - compiled_at=datetime.now(timezone.utc), - created_at=datetime.now(timezone.utc), + compiled_at=datetime.now(UTC), + created_at=datetime.now(UTC), ) session.previews = [latest_preview] repository = MagicMock() @@ -983,8 +983,8 @@ def test_us3_mapping_patch_approval_preview_and_launch_endpoints( compiled_by="superset", error_code=None, error_details=None, - compiled_at=datetime.now(timezone.utc), - created_at=datetime.now(timezone.utc), + compiled_at=datetime.now(UTC), + created_at=datetime.now(UTC), ) run_context = SimpleNamespace( run_context_id="run-1", @@ -1000,7 +1000,7 @@ def test_us3_mapping_patch_approval_preview_and_launch_endpoints( open_warning_refs=[], launch_status=LaunchStatus.STARTED, launch_error=None, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) orchestrator = MagicMock() orchestrator.prepare_launch_preview.return_value = PreparePreviewResult( @@ -1099,7 +1099,7 @@ def test_us3_mapping_patch_approval_preview_and_launch_endpoints( error_code=None, error_details=None, compiled_at=None, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ), blocked_reasons=[], ) @@ -1161,8 +1161,8 @@ def test_us3_preview_response_propagates_refreshed_session_version_for_launch_fo compiled_by="superset", error_code=None, error_details=None, - compiled_at=datetime.now(timezone.utc), - created_at=datetime.now(timezone.utc), + compiled_at=datetime.now(UTC), + created_at=datetime.now(UTC), ) run_context = SimpleNamespace( run_context_id="run-2", @@ -1178,7 +1178,7 @@ def test_us3_preview_response_propagates_refreshed_session_version_for_launch_fo open_warning_refs=[], launch_status=LaunchStatus.STARTED, launch_error=None, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) orchestrator = MagicMock() def _prepare_preview(command): @@ -1235,7 +1235,7 @@ def test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not error_code="superset_preview_failed", error_details="RuntimeError: [API_FAILURE] API resource not found at endpoint '/chart/data' | Context: {'status_code': 404, 'endpoint': '/chart/data', 'subtype': 'not_found'}", compiled_at=None, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) orchestrator = MagicMock() orchestrator.prepare_launch_preview.return_value = PreparePreviewResult( @@ -1479,7 +1479,7 @@ def test_us3_launch_endpoint_requires_launch_permission( open_warning_refs=[], launch_status=LaunchStatus.STARTED, launch_error=None, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) orchestrator = MagicMock() orchestrator.launch_dataset.return_value = LaunchDatasetResult( @@ -1534,4 +1534,4 @@ def test_semantic_source_version_propagation_preserves_locked_fields(): assert locked_field.source_version == "2026.03" assert locked_field.needs_review is False # #endregion test_semantic_source_version_propagation_preserves_locked_fields -# #endregion DatasetReviewApiTests \ No newline at end of file +# #endregion DatasetReviewApiTests diff --git a/backend/src/api/routes/__tests__/test_datasets.py b/backend/src/api/routes/__tests__/test_datasets.py index 57d0f530..e3bbffa8 100644 --- a/backend/src/api/routes/__tests__/test_datasets.py +++ b/backend/src/api/routes/__tests__/test_datasets.py @@ -3,19 +3,22 @@ # @LAYER: API # @RELATION DEPENDS_ON -> [DatasetsApi] # @INVARIANT: Endpoint contracts remain stable for success and validation failure paths. +from unittest.mock import AsyncMock, MagicMock + import pytest -from unittest.mock import MagicMock, patch, AsyncMock from fastapi.testclient import TestClient + +from src.api.routes.datasets import DatasetsResponse from src.app import app -from src.api.routes.datasets import DatasetsResponse, DatasetDetailResponse from src.dependencies import ( - get_current_user, - has_permission, get_config_manager, - get_task_manager, - get_resource_service, + get_current_user, get_mapping_service, + get_resource_service, + get_task_manager, + has_permission, ) + # Global mock user for get_current_user dependency overrides mock_user = MagicMock() mock_user.username = "testuser" @@ -253,4 +256,4 @@ def test_get_datasets_superset_failure(mock_deps): assert response.status_code == 503 assert "Failed to fetch datasets" in response.json()["detail"] # #endregion test_get_datasets_superset_failure -# #endregion DatasetsApiTests \ No newline at end of file +# #endregion DatasetsApiTests diff --git a/backend/src/api/routes/__tests__/test_git_api.py b/backend/src/api/routes/__tests__/test_git_api.py index 1c7408da..bb0a16be 100644 --- a/backend/src/api/routes/__tests__/test_git_api.py +++ b/backend/src/api/routes/__tests__/test_git_api.py @@ -1,12 +1,16 @@ # #region TestGitApi [TYPE Module] [C:3] [SEMANTICS test, git, api, config, repository] # @RELATION VERIFIES -> [GitApi] # @BRIEF API tests for Git configurations and repository operations. -import pytest import asyncio from unittest.mock import MagicMock + +import pytest from fastapi import HTTPException + from src.api.routes import git as git_routes -from src.models.git import GitServerConfig, GitProvider, GitStatus, GitRepository +from src.models.git import GitProvider, GitRepository, GitServerConfig, GitStatus + + # #region DbMock [TYPE Class] [C:2] # @RELATION BINDS_TO -> [TestGitApi] # @BRIEF In-memory session double for git route tests with minimal query/filter persistence semantics. @@ -123,6 +127,8 @@ def test_create_git_config_persists_config(): ) # Note: route returns unmasked until serialized by FastAPI usually, but in tests schema might catch it or not. # #endregion test_create_git_config_persists_config from src.api.routes.git_schemas import GitServerConfigUpdate + + # #region test_update_git_config_modifies_record [TYPE Function] # @RELATION BINDS_TO -> [TestGitApi] # @BRIEF Validate updating git config modifies mutable fields while preserving masked PAT semantics. @@ -406,4 +412,4 @@ def test_init_repository_initializes_and_saves_binding(monkeypatch): assert isinstance(db._added[0], GitRepository) assert db._added[0].dashboard_id == 123 # #endregion test_init_repository_initializes_and_saves_binding -# #endregion TestGitApi \ No newline at end of file +# #endregion TestGitApi diff --git a/backend/src/api/routes/__tests__/test_git_status_route.py b/backend/src/api/routes/__tests__/test_git_status_route.py index 210fcf2e..506c590d 100644 --- a/backend/src/api/routes/__tests__/test_git_status_route.py +++ b/backend/src/api/routes/__tests__/test_git_status_route.py @@ -2,11 +2,15 @@ # @BRIEF Validate status endpoint behavior for missing and error repository states. # @LAYER: Domain (Tests) # @RELATION VERIFIES -> [GitApi] -from fastapi import HTTPException -import pytest import asyncio from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException + from src.api.routes import git as git_routes + + # #region test_get_repository_status_returns_no_repo_payload_for_missing_repo [TYPE Function] # @RELATION BINDS_TO -> TestGitStatusRoute # @BRIEF Ensure missing local repository is represented as NO_REPO payload instead of an API error. @@ -347,4 +351,4 @@ def test_continue_merge_passes_message_and_returns_commit(monkeypatch): assert response["status"] == "committed" assert response["commit_hash"] == "abc123" # #endregion test_continue_merge_passes_message_and_returns_commit -# #endregion TestGitStatusRoute \ No newline at end of file +# #endregion TestGitStatusRoute diff --git a/backend/src/api/routes/__tests__/test_migration_routes.py b/backend/src/api/routes/__tests__/test_migration_routes.py index a4214720..28a2ee07 100644 --- a/backend/src/api/routes/__tests__/test_migration_routes.py +++ b/backend/src/api/routes/__tests__/test_migration_routes.py @@ -4,16 +4,19 @@ # @LAYER: API # @RELATION VERIFIES -> backend.src.api.routes.migration # -import pytest import sys +from datetime import UTC, datetime from pathlib import Path -from unittest.mock import MagicMock, AsyncMock, patch -from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + # Add backend directory to sys.path backend_dir = str(Path(__file__).parent.parent.parent.parent.resolve()) if backend_dir not in sys.path: sys.path.insert(0, backend_dir) import os + # Force SQLite in-memory for all database connections BEFORE importing any application code # @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv. os.environ["DATABASE_URL"] = "sqlite:///:memory:" @@ -26,9 +29,10 @@ os.environ["ENVIRONMENT"] = "testing" from fastapi import HTTPException from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker + from src.models.mapping import Base, ResourceMapping, ResourceType + # Patch the get_db dependency if `src.api.routes.migration` imports it -from unittest.mock import patch patch("src.core.database.get_db").start() # --- Fixtures --- @pytest.fixture @@ -111,7 +115,7 @@ async def test_get_resource_mappings_returns_formatted_list(db_session): uuid="uuid-1", remote_integer_id="42", resource_name="Sales Chart", - last_synced_at=datetime(2026, 1, 15, 12, 0, 0, tzinfo=timezone.utc), + last_synced_at=datetime(2026, 1, 15, 12, 0, 0, tzinfo=UTC), ) db_session.add(m1) db_session.commit() @@ -517,4 +521,4 @@ async def test_dry_run_migration_rejects_same_environment(db_session): selection=selection, config_manager=cm, db=db_session, _=None ) assert exc.value.status_code == 400 -# #endregion TestMigrationRoutes \ No newline at end of file +# #endregion TestMigrationRoutes diff --git a/backend/src/api/routes/__tests__/test_profile_api.py b/backend/src/api/routes/__tests__/test_profile_api.py index 881b62d4..5cfc697e 100644 --- a/backend/src/api/routes/__tests__/test_profile_api.py +++ b/backend/src/api/routes/__tests__/test_profile_api.py @@ -3,9 +3,11 @@ # @BRIEF Verifies profile API route contracts for preference read/update and Superset account lookup. # @LAYER: API # [SECTION: IMPORTS] -from datetime import datetime, timezone +from datetime import UTC, datetime from unittest.mock import MagicMock, patch + from fastapi.testclient import TestClient + from src.app import app from src.core.database import get_db from src.dependencies import get_config_manager, get_current_user @@ -22,6 +24,7 @@ from src.services.profile_service import ( ProfileAuthorizationError, ProfileValidationError, ) + # [/SECTION] client = TestClient(app) # #region mock_profile_route_dependencies [TYPE Function] @@ -46,6 +49,8 @@ def mock_profile_route_dependencies(): # @PRE: None. # @POST: Yields overridden dependencies and clears overrides after test. import pytest + + @pytest.fixture(autouse=True) def profile_route_deps_fixture(): yielded = mock_profile_route_dependencies() @@ -58,7 +63,7 @@ def profile_route_deps_fixture(): # @PRE: user_id is provided. # @POST: Returns ProfilePreferenceResponse object with deterministic timestamps. def _build_preference_response(user_id: str = "u-1") -> ProfilePreferenceResponse: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) return ProfilePreferenceResponse( status="success", message="Preference loaded", @@ -260,4 +265,4 @@ def test_lookup_superset_accounts_env_not_found(profile_route_deps_fixture): payload = response.json() assert payload["detail"] == "Environment 'missing-env' not found" # #endregion test_lookup_superset_accounts_env_not_found -# #endregion TestProfileApi \ No newline at end of file +# #endregion TestProfileApi diff --git a/backend/src/api/routes/__tests__/test_reports_api.py b/backend/src/api/routes/__tests__/test_reports_api.py index c821d5df..c2070188 100644 --- a/backend/src/api/routes/__tests__/test_reports_api.py +++ b/backend/src/api/routes/__tests__/test_reports_api.py @@ -3,12 +3,16 @@ # @BRIEF Contract tests for GET /api/reports defaults, pagination, and filtering behavior. # @LAYER: Domain (Tests) # @INVARIANT: API response contract contains {items,total,page,page_size,has_next,applied_filters}. -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from types import SimpleNamespace + from fastapi.testclient import TestClient + from src.app import app from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager + + # @DEBT: Divergent _FakeTaskManager definition. Canonical version should be in conftest.py. Authz variant is missing get_all_tasks(). # #region _FakeTaskManager [TYPE Class] [C:1] # @RELATION BINDS_TO -> [TestReportsApi] @@ -147,7 +151,7 @@ def test_get_reports_filter_and_pagination(): # @BRIEF Validate reports sorting remains stable when task timestamps mix naive and timezone-aware datetimes. def test_get_reports_handles_mixed_naive_and_aware_datetimes(): naive_now = datetime.utcnow() - aware_now = datetime.now(timezone.utc) + aware_now = datetime.now(UTC) tasks = [ _make_task( "t-naive", @@ -201,4 +205,4 @@ def test_get_reports_invalid_filter_returns_400(): finally: app.dependency_overrides.clear() # #endregion test_get_reports_invalid_filter_returns_400 -# #endregion TestReportsApi \ No newline at end of file +# #endregion TestReportsApi diff --git a/backend/src/api/routes/__tests__/test_reports_detail_api.py b/backend/src/api/routes/__tests__/test_reports_detail_api.py index 9b0d4d0a..3f387002 100644 --- a/backend/src/api/routes/__tests__/test_reports_detail_api.py +++ b/backend/src/api/routes/__tests__/test_reports_detail_api.py @@ -5,10 +5,14 @@ # @INVARIANT: Detail endpoint tests must keep deterministic assertions for success and not-found contracts. from datetime import datetime, timedelta from types import SimpleNamespace + from fastapi.testclient import TestClient + from src.app import app from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager + + # @DEBT: Divergent _FakeTaskManager definition. Canonical version should be in conftest.py. Authz variant is missing get_all_tasks(). # #region _FakeTaskManager [TYPE Class] [C:1] # @RELATION BINDS_TO -> [TestReportsDetailApi] @@ -87,4 +91,4 @@ def test_get_report_detail_not_found(): finally: app.dependency_overrides.clear() # #endregion test_get_report_detail_not_found -# #endregion TestReportsDetailApi \ No newline at end of file +# #endregion TestReportsDetailApi diff --git a/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py b/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py index 91e2273a..b7fc8619 100644 --- a/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py +++ b/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py @@ -5,10 +5,14 @@ # @INVARIANT: List and detail payloads include required contract keys. from datetime import datetime from types import SimpleNamespace + from fastapi.testclient import TestClient + from src.app import app from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager + + # #region _FakeTaskManager [TYPE Class] [C:1] # @RELATION BINDS_TO -> [TestReportsOpenapiConformance] # @BRIEF Minimal task-manager fake exposing static task list for OpenAPI conformance checks. @@ -84,4 +88,4 @@ def test_reports_detail_openapi_required_keys(): finally: app.dependency_overrides.clear() # #endregion test_reports_detail_openapi_required_keys -# #endregion TestReportsOpenapiConformance \ No newline at end of file +# #endregion TestReportsOpenapiConformance diff --git a/backend/src/api/routes/__tests__/test_tasks_logs.py b/backend/src/api/routes/__tests__/test_tasks_logs.py index 6875635f..a4e836d6 100644 --- a/backend/src/api/routes/__tests__/test_tasks_logs.py +++ b/backend/src/api/routes/__tests__/test_tasks_logs.py @@ -2,12 +2,16 @@ # @RELATION VERIFIES -> [src.api.routes.tasks:Module] # @BRIEF Contract testing for task logs API endpoints. # @LAYER: Domain (Tests) +from unittest.mock import MagicMock + import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -from unittest.mock import MagicMock -from src.dependencies import get_task_manager, has_permission + from src.api.routes.tasks import router +from src.dependencies import get_task_manager, has_permission + + # @TEST_FIXTURE: mock_app @pytest.fixture def client(): @@ -74,4 +78,4 @@ def test_get_task_log_stats_success(client): # response_model=LogStats might wrap this, but let's check basic structure # assuming tm.get_task_log_stats returns something compatible with LogStats # #endregion test_get_task_log_stats_success -# #endregion test_tasks_logs_module \ No newline at end of file +# #endregion test_tasks_logs_module diff --git a/backend/src/api/routes/admin.py b/backend/src/api/routes/admin.py index e71d759c..b2aa47d0 100644 --- a/backend/src/api/routes/admin.py +++ b/backend/src/api/routes/admin.py @@ -8,26 +8,29 @@ # # @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope. -from typing import List + from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session -from ...core.database import get_auth_db + from ...core.auth.repository import AuthRepository from ...core.auth.security import get_password_hash +from ...core.database import get_auth_db +from ...core.logger import belief_scope, logger +from ...dependencies import get_plugin_loader, has_permission +from ...models.auth import ADGroupMapping, Role, User from ...schemas.auth import ( - User as UserSchema, + ADGroupMappingCreate, + ADGroupMappingSchema, + PermissionSchema, + RoleCreate, + RoleSchema, + RoleUpdate, UserCreate, UserUpdate, - RoleSchema, - RoleCreate, - RoleUpdate, - PermissionSchema, - ADGroupMappingSchema, - ADGroupMappingCreate, ) -from ...models.auth import User, Role, ADGroupMapping -from ...dependencies import has_permission, get_plugin_loader -from ...core.logger import logger, belief_scope +from ...schemas.auth import ( + User as UserSchema, +) from ...services.rbac_permission_catalog import ( discover_declared_permissions, sync_permission_catalog, @@ -45,7 +48,7 @@ router = APIRouter(prefix="/api/admin", tags=["admin"]) # @PRE: Current user has 'Admin' role. # @POST: Returns a list of UserSchema objects. # @RELATION CALLS -> User -@router.get("/users", response_model=List[UserSchema]) +@router.get("/users", response_model=list[UserSchema]) async def list_users( db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:users", "READ")) ): @@ -175,7 +178,7 @@ async def delete_user( # #region list_roles [C:3] [TYPE Function] # @BRIEF Lists all available roles. # @RELATION CALLS -> [Role:Class] -@router.get("/roles", response_model=List[RoleSchema]) +@router.get("/roles", response_model=list[RoleSchema]) async def list_roles( db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:roles", "READ")) ): @@ -296,7 +299,7 @@ async def delete_role( # @BRIEF Lists all available system permissions for assignment. # @POST: Returns a list of all PermissionSchema objects. # @RELATION CALLS -> backend.src.core.auth.repository.AuthRepository.list_permissions -@router.get("/permissions", response_model=List[PermissionSchema]) +@router.get("/permissions", response_model=list[PermissionSchema]) async def list_permissions( db: Session = Depends(get_auth_db), plugin_loader=Depends(get_plugin_loader), @@ -325,7 +328,7 @@ async def list_permissions( # #region list_ad_mappings [C:3] [TYPE Function] # @BRIEF Lists all AD Group to Role mappings. # @RELATION CALLS -> ADGroupMapping -@router.get("/ad-mappings", response_model=List[ADGroupMappingSchema]) +@router.get("/ad-mappings", response_model=list[ADGroupMappingSchema]) async def list_ad_mappings( db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:settings", "READ")), diff --git a/backend/src/api/routes/assistant/__init__.py b/backend/src/api/routes/assistant/__init__.py index 29912f3a..3e08ec7c 100644 --- a/backend/src/api/routes/assistant/__init__.py +++ b/backend/src/api/routes/assistant/__init__.py @@ -8,107 +8,107 @@ # @INVARIANT: Risky operations are never executed without valid confirmation token. # Re-export public API for backward compatibility. -from ._routes import router, send_message, confirm_operation, cancel_operation -from ._admin_routes import list_conversations, delete_conversation, get_history, get_assistant_audit -from ._schemas import ( - AssistantMessageRequest, - AssistantMessageResponse, - AssistantAction, - ConfirmationRecord, - CONVERSATIONS, - USER_ACTIVE_CONVERSATION, - CONFIRMATIONS, - ASSISTANT_AUDIT, - INTENT_PERMISSION_CHECKS, - _SAFE_OPS, - _DATASET_REVIEW_OPS, -) +from ._admin_routes import delete_conversation, get_assistant_audit, get_history, list_conversations from ._command_parser import _parse_command -from ._llm_planner import _build_tool_catalog -from ._llm_planner_intent import _plan_intent_with_llm, _authorize_intent -from ._dispatch import _dispatch_intent, _async_confirmation_summary, _clarification_text_for_intent from ._dataset_review import ( _load_dataset_review_context, _plan_dataset_review_intent, ) from ._dataset_review_dispatch import _dispatch_dataset_review_intent +from ._dispatch import _async_confirmation_summary, _clarification_text_for_intent, _dispatch_intent from ._history import ( _append_history, - _persist_message, _audit, + _cleanup_history_ttl, + _coerce_query_bool, + _is_conversation_archived, + _load_confirmation_from_db, _persist_audit, _persist_confirmation, - _update_confirmation_state, - _load_confirmation_from_db, + _persist_message, _resolve_or_create_conversation, - _cleanup_history_ttl, - _is_conversation_archived, - _coerce_query_bool, + _update_confirmation_state, ) +from ._llm_planner import _build_tool_catalog +from ._llm_planner_intent import _authorize_intent, _plan_intent_with_llm from ._resolvers import ( + _build_task_observability_summary, _extract_id, - _resolve_env_id, - _is_production_env, - _resolve_provider_id, + _extract_result_deep_links, _get_default_environment_id, + _get_environment_name_by_id, + _is_production_env, _resolve_dashboard_id_by_ref, _resolve_dashboard_id_entity, - _get_environment_name_by_id, - _extract_result_deep_links, - _build_task_observability_summary, + _resolve_env_id, + _resolve_provider_id, +) +from ._routes import cancel_operation, confirm_operation, router, send_message +from ._schemas import ( + _DATASET_REVIEW_OPS, + _SAFE_OPS, + ASSISTANT_AUDIT, + CONFIRMATIONS, + CONVERSATIONS, + INTENT_PERMISSION_CHECKS, + USER_ACTIVE_CONVERSATION, + AssistantAction, + AssistantMessageRequest, + AssistantMessageResponse, + ConfirmationRecord, ) __all__ = [ - "router", - "send_message", - "confirm_operation", - "cancel_operation", - "list_conversations", - "delete_conversation", - "get_history", - "get_assistant_audit", + "ASSISTANT_AUDIT", + "CONFIRMATIONS", + "CONVERSATIONS", + "INTENT_PERMISSION_CHECKS", + "USER_ACTIVE_CONVERSATION", + "_DATASET_REVIEW_OPS", + "_SAFE_OPS", + "AssistantAction", "AssistantMessageRequest", "AssistantMessageResponse", - "AssistantAction", "ConfirmationRecord", - "CONVERSATIONS", - "USER_ACTIVE_CONVERSATION", - "CONFIRMATIONS", - "ASSISTANT_AUDIT", - "INTENT_PERMISSION_CHECKS", - "_SAFE_OPS", - "_DATASET_REVIEW_OPS", - "_parse_command", - "_plan_intent_with_llm", - "_build_tool_catalog", - "_authorize_intent", - "_dispatch_intent", - "_async_confirmation_summary", - "_clarification_text_for_intent", - "_load_dataset_review_context", - "_plan_dataset_review_intent", - "_dispatch_dataset_review_intent", "_append_history", - "_persist_message", + "_async_confirmation_summary", "_audit", + "_authorize_intent", + "_build_task_observability_summary", + "_build_tool_catalog", + "_clarification_text_for_intent", + "_cleanup_history_ttl", + "_coerce_query_bool", + "_dispatch_dataset_review_intent", + "_dispatch_intent", + "_extract_id", + "_extract_result_deep_links", + "_get_default_environment_id", + "_get_environment_name_by_id", + "_is_conversation_archived", + "_is_production_env", + "_load_confirmation_from_db", + "_load_dataset_review_context", + "_parse_command", "_persist_audit", "_persist_confirmation", - "_update_confirmation_state", - "_load_confirmation_from_db", - "_resolve_or_create_conversation", - "_cleanup_history_ttl", - "_is_conversation_archived", - "_coerce_query_bool", - "_extract_id", - "_resolve_env_id", - "_is_production_env", - "_resolve_provider_id", - "_get_default_environment_id", + "_persist_message", + "_plan_dataset_review_intent", + "_plan_intent_with_llm", "_resolve_dashboard_id_by_ref", "_resolve_dashboard_id_entity", - "_get_environment_name_by_id", - "_extract_result_deep_links", - "_build_task_observability_summary", + "_resolve_env_id", + "_resolve_or_create_conversation", + "_resolve_provider_id", + "_update_confirmation_state", + "cancel_operation", + "confirm_operation", + "delete_conversation", + "get_assistant_audit", + "get_history", + "list_conversations", + "router", + "send_message", ] # #endregion AssistantApi diff --git a/backend/src/api/routes/assistant/_admin_routes.py b/backend/src/api/routes/assistant/_admin_routes.py index ec5a1c50..5db823da 100644 --- a/backend/src/api/routes/assistant/_admin_routes.py +++ b/backend/src/api/routes/assistant/_admin_routes.py @@ -8,36 +8,33 @@ from __future__ import annotations -import uuid from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any from fastapi import Depends, HTTPException, Query -from sqlalchemy.orm import Session from sqlalchemy import desc +from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger -from src.dependencies import get_current_user, has_permission from src.core.database import get_db +from src.core.logger import belief_scope +from src.dependencies import get_current_user, has_permission from src.models.assistant import ( AssistantAuditRecord, AssistantMessageRecord, ) from src.schemas.auth import User -from ._schemas import ( - ASSISTANT_AUDIT, - CONVERSATIONS, - AssistantAction, - AssistantMessageResponse, -) + from ._history import ( _cleanup_history_ttl, _coerce_query_bool, _is_conversation_archived, - _persist_audit, _resolve_or_create_conversation, ) from ._routes import router +from ._schemas import ( + ASSISTANT_AUDIT, + CONVERSATIONS, +) # #region list_conversations [C:2] [TYPE Function] @@ -50,7 +47,7 @@ async def list_conversations( page_size: int = Query(20, ge=1, le=100), include_archived: bool = Query(False), archived_only: bool = Query(False), - search: Optional[str] = Query(None), + search: str | None = Query(None), current_user: User = Depends(get_current_user), db: Session = Depends(get_db), ): @@ -67,7 +64,7 @@ async def list_conversations( .all() ) - summary: Dict[str, Dict[str, Any]] = {} + summary: dict[str, dict[str, Any]] = {} for row in rows: conv_id = row.conversation_id if not conv_id: @@ -189,7 +186,7 @@ async def delete_conversation( async def get_history( page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100), - conversation_id: Optional[str] = Query(None), + conversation_id: str | None = Query(None), from_latest: bool = Query(False), current_user: User = Depends(get_current_user), db: Session = Depends(get_db), diff --git a/backend/src/api/routes/assistant/_command_parser.py b/backend/src/api/routes/assistant/_command_parser.py index f8d5d1c3..5a18d6c4 100644 --- a/backend/src/api/routes/assistant/_command_parser.py +++ b/backend/src/api/routes/assistant/_command_parser.py @@ -7,10 +7,11 @@ from __future__ import annotations import re -from typing import Any, Dict, List, Optional +from typing import Any -from src.core.logger import belief_scope, logger from src.core.config_manager import ConfigManager +from src.core.logger import belief_scope, logger + from ._resolvers import _extract_id, _is_production_env @@ -23,36 +24,36 @@ from ._resolvers import _extract_id, _is_production_env # @PRE: message contains raw user text and config manager resolves environments. # @POST: Returns intent dict with domain/operation/entities/confidence/risk fields. # @INVARIANT: every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. -def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any]: +def _parse_command(message: str, config_manager: ConfigManager) -> dict[str, Any]: with belief_scope('_parse_command'): logger.reason('Belief protocol reasoning checkpoint for _parse_command') text = message.strip() lower = text.lower() - if any((phrase in lower for phrase in ['что ты умеешь', 'что умеешь', 'что ты можешь', 'help', 'помощь', 'доступные команды', 'какие команды'])): + if any(phrase in lower for phrase in ['что ты умеешь', 'что умеешь', 'что ты можешь', 'help', 'помощь', 'доступные команды', 'какие команды']): logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'assistant', 'operation': 'show_capabilities', 'entities': {}, 'confidence': 0.98, 'risk_level': 'safe', 'requires_confirmation': False} dashboard_id = _extract_id(lower, ['(?:дашборд\\w*|dashboard)\\s*(?:id\\s*)?(\\d+)']) dashboard_ref = _extract_id(lower, ['(?:дашборд\\w*|dashboard)\\s*(?:id\\s*)?([a-zа-я0-9._-]+)']) dataset_id = _extract_id(lower, ['(?:датасет\\w*|dataset)\\s*(?:id\\s*)?(\\d+)']) task_id = _extract_id(lower, ['(task[-_a-z0-9]{1,}|[0-9a-f]{8}-[0-9a-f-]{27,})']) - if any((k in lower for k in ['статус', 'status', 'state', 'проверь задачу'])): + if any(k in lower for k in ['статус', 'status', 'state', 'проверь задачу']): logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'status', 'operation': 'get_task_status', 'entities': {'task_id': task_id}, 'confidence': 0.92 if task_id else 0.66, 'risk_level': 'safe', 'requires_confirmation': False} - if any((k in lower for k in ['ветк', 'branch'])) and any((k in lower for k in ['созд', 'сделай', 'create'])): + if any(k in lower for k in ['ветк', 'branch']) and any(k in lower for k in ['созд', 'сделай', 'create']): branch = _extract_id(lower, ['(?:ветк\\w*|branch)\\s+([a-z0-9._/-]+)']) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'git', 'operation': 'create_branch', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'branch_name': branch}, 'confidence': 0.95 if branch and dashboard_id else 0.7, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['коммит', 'commit'])): + if any(k in lower for k in ['коммит', 'commit']): quoted = re.search('"([^"]{3,120})"', text) message_text = quoted.group(1) if quoted else 'assistant: update dashboard changes' logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'git', 'operation': 'commit_changes', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'message': message_text}, 'confidence': 0.9 if dashboard_id else 0.7, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['деплой', 'deploy', 'разверн'])): + if any(k in lower for k in ['деплой', 'deploy', 'разверн']): env_match = _extract_id(lower, ['(?:в|to)\\s+([a-z0-9_-]+)']) is_dangerous = _is_production_env(env_match, config_manager) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'git', 'operation': 'deploy_dashboard', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'environment': env_match}, 'confidence': 0.92 if dashboard_id and env_match else 0.7, 'risk_level': 'dangerous' if is_dangerous else 'guarded', 'requires_confirmation': is_dangerous} - if any((k in lower for k in ['миграц', 'migration', 'migrate'])): + if any(k in lower for k in ['миграц', 'migration', 'migrate']): src = _extract_id(lower, ['(?:с|from)\\s+([a-z0-9_-]+)']) tgt = _extract_id(lower, ['(?:на|to)\\s+([a-z0-9_-]+)']) dry_run = '--dry-run' in lower or 'dry run' in lower @@ -61,20 +62,20 @@ def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any is_dangerous = _is_production_env(tgt, config_manager) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'migration', 'operation': 'execute_migration', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'source_env': src, 'target_env': tgt, 'dry_run': dry_run, 'replace_db_config': replace_db_config, 'fix_cross_filters': fix_cross_filters}, 'confidence': 0.95 if dashboard_id and src and tgt else 0.72, 'risk_level': 'dangerous' if is_dangerous else 'guarded', 'requires_confirmation': is_dangerous or dry_run} - if any((k in lower for k in ['бэкап', 'backup', 'резерв'])): + if any(k in lower for k in ['бэкап', 'backup', 'резерв']): env_match = _extract_id(lower, ['(?:в|for|из|from)\\s+([a-z0-9_-]+)']) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'backup', 'operation': 'run_backup', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'environment': env_match}, 'confidence': 0.9 if env_match else 0.7, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['здоровье', 'health', 'ошибки', 'failing', 'проблемы'])): + if any(k in lower for k in ['здоровье', 'health', 'ошибки', 'failing', 'проблемы']): env_match = _extract_id(lower, ['(?:в|for|env|окружени[ея])\\s+([a-z0-9_-]+)']) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'health', 'operation': 'get_health_summary', 'entities': {'environment': env_match}, 'confidence': 0.9, 'risk_level': 'safe', 'requires_confirmation': False} - if any((k in lower for k in ['валидац', 'validate', 'провер'])): + if any(k in lower for k in ['валидац', 'validate', 'провер']): env_match = _extract_id(lower, ['(?:в|for|env|окружени[ея])\\s+([a-z0-9_-]+)']) provider_match = _extract_id(lower, ['(?:provider|провайдер)\\s+([a-z0-9_-]+)']) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') return {'domain': 'llm', 'operation': 'run_llm_validation', 'entities': {'dashboard_id': int(dashboard_id) if dashboard_id else None, 'dashboard_ref': dashboard_ref if dashboard_ref and (not dashboard_ref.isdigit()) else None, 'environment': env_match, 'provider': provider_match}, 'confidence': 0.88 if dashboard_id else 0.64, 'risk_level': 'guarded', 'requires_confirmation': False} - if any((k in lower for k in ['документац', 'documentation', 'generate docs', 'сгенерируй док'])): + if any(k in lower for k in ['документац', 'documentation', 'generate docs', 'сгенерируй док']): env_match = _extract_id(lower, ['(?:в|for|env|окружени[ея])\\s+([a-z0-9_-]+)']) provider_match = _extract_id(lower, ['(?:provider|провайдер)\\s+([a-z0-9_-]+)']) logger.reflect('Belief protocol postcondition checkpoint for _parse_command') diff --git a/backend/src/api/routes/assistant/_dataset_review.py b/backend/src/api/routes/assistant/_dataset_review.py index 077c15d9..0b7d1825 100644 --- a/backend/src/api/routes/assistant/_dataset_review.py +++ b/backend/src/api/routes/assistant/_dataset_review.py @@ -9,10 +9,9 @@ from __future__ import annotations import re -from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple +from typing import Any -from fastapi import HTTPException, status +from fastapi import HTTPException from sqlalchemy.orm import Session from src.core.logger import belief_scope, logger @@ -22,16 +21,10 @@ from src.core.utils.superset_context_extractor import ( from src.models.dataset_review import ( ApprovalState, DatasetReviewSession, - ReadinessState, - RecommendedAction, -) -from src.services.dataset_review.repositories.session_repository import ( - DatasetReviewSessionRepository, - DatasetReviewSessionVersionConflictError, ) from src.schemas.auth import User -from ._schemas import ( - AssistantAction, +from src.services.dataset_review.repositories.session_repository import ( + DatasetReviewSessionRepository, ) @@ -41,7 +34,7 @@ from ._schemas import ( # @PRE: session_id is a valid active review session identifier. # @POST: Returns a serializable dictionary containing the complete review context. # @SIDE_EFFECT: Reads session data from the database. -def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str, Any]: +def _serialize_dataset_review_context(session: DatasetReviewSession) -> dict[str, Any]: with belief_scope('_serialize_dataset_review_context'): logger.reason('Belief protocol reasoning checkpoint for _serialize_dataset_review_context') latest_preview = None @@ -61,7 +54,7 @@ def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str # @PRE: session_id is a valid active review session identifier. # @POST: Returns a loaded context object with session data and findings. # @SIDE_EFFECT: Reads session data from the database. -def _load_dataset_review_context(dataset_review_session_id: Optional[str], current_user: User, db: Session) -> Optional[Dict[str, Any]]: +def _load_dataset_review_context(dataset_review_session_id: str | None, current_user: User, db: Session) -> dict[str, Any] | None: with belief_scope('_load_dataset_review_context'): if not dataset_review_session_id: return None @@ -79,7 +72,7 @@ def _load_dataset_review_context(dataset_review_session_id: Optional[str], curre # #region _extract_dataset_review_target [C:2] [TYPE Function] # @BRIEF Extract structured dataset-review focus target hints embedded in assistant prompts. -def _extract_dataset_review_target(message: str) -> Tuple[Optional[str], Optional[str]]: +def _extract_dataset_review_target(message: str) -> tuple[str | None, str | None]: match = re.search( r"(?:target|focus)\s*[:=]\s*(field|mapping|finding|filter)[:=]([A-Za-z0-9._-]+)", str(message or ""), @@ -96,9 +89,9 @@ def _extract_dataset_review_target(message: str) -> Tuple[Optional[str], Optiona # #region _match_dataset_review_field [C:2] [TYPE Function] # @BRIEF Resolve one semantic field from assistant-visible context by id or user-visible label. def _match_dataset_review_field( - dataset_context: Dict[str, Any], + dataset_context: dict[str, Any], message: str, -) -> Optional[Dict[str, Any]]: +) -> dict[str, Any] | None: target_kind, target_id = _extract_dataset_review_target(message) fields = dataset_context.get("semantic_fields", []) or [] if target_kind == "field" and target_id: @@ -125,7 +118,7 @@ def _match_dataset_review_field( # #region _extract_quoted_segment [C:2] [TYPE Function] # @BRIEF Extract one quoted assistant command segment after a label token. -def _extract_quoted_segment(message: str, label: str) -> Optional[str]: +def _extract_quoted_segment(message: str, label: str) -> str | None: pattern = rf"{label}\s*[=:]?\s*[\"']([^\"']+)[\"']" match = re.search(pattern, str(message or ""), re.IGNORECASE) return match.group(1).strip() if match else None @@ -139,8 +132,8 @@ def _extract_quoted_segment(message: str, label: str) -> Optional[str]: # @RELATION CALLS -> DatasetReviewOrchestrator def _plan_dataset_review_intent( message: str, - dataset_context: Dict[str, Any], -) -> Optional[Dict[str, Any]]: + dataset_context: dict[str, Any], +) -> dict[str, Any] | None: lower = message.strip().lower() session_id = dataset_context["session_id"] session_version = int(dataset_context.get("version", 0) or 0) diff --git a/backend/src/api/routes/assistant/_dataset_review_dispatch.py b/backend/src/api/routes/assistant/_dataset_review_dispatch.py index 54677f3c..bc6d6ea0 100644 --- a/backend/src/api/routes/assistant/_dataset_review_dispatch.py +++ b/backend/src/api/routes/assistant/_dataset_review_dispatch.py @@ -9,18 +9,20 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from fastapi import HTTPException, status from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger +from src.api.routes.dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state from src.core.config_manager import ConfigManager +from src.core.logger import belief_scope, logger from src.models.dataset_review import ( ApprovalState, ReadinessState, RecommendedAction, ) +from src.schemas.auth import User from src.services.dataset_review.orchestrator import ( DatasetReviewOrchestrator, PreparePreviewCommand, @@ -29,8 +31,7 @@ from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionRepository, DatasetReviewSessionVersionConflictError, ) -from src.schemas.auth import User -from src.api.routes.dataset_review import FieldSemanticUpdateRequest, _update_semantic_field_state + from ._schemas import ( AssistantAction, ) @@ -63,11 +64,11 @@ def _dataset_review_conflict_http_exception( # @POST: Returns a structured response with planned actions and confirmations. # @SIDE_EFFECT: May update session state and enqueue tasks. async def _dispatch_dataset_review_intent( - intent: Dict[str, Any], + intent: dict[str, Any], current_user: User, config_manager: ConfigManager, db: Session, -) -> Tuple[str, Optional[str], List[AssistantAction]]: +) -> tuple[str, str | None, list[AssistantAction]]: with belief_scope("_dispatch_dataset_review_intent"): logger.reason( "Dispatching assistant dataset-review intent", diff --git a/backend/src/api/routes/assistant/_dispatch.py b/backend/src/api/routes/assistant/_dispatch.py index 94b22887..c0dea52b 100644 --- a/backend/src/api/routes/assistant/_dispatch.py +++ b/backend/src/api/routes/assistant/_dispatch.py @@ -9,22 +9,22 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from fastapi import HTTPException from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger from src.core.config_manager import ConfigManager +from src.core.logger import belief_scope, logger from src.core.task_manager import TaskManager from src.schemas.auth import User from src.services.git_service import GitService -from src.services.llm_provider import LLMProviderService from src.services.llm_prompt_templates import is_multimodal_model -from ._schemas import ( - _DATASET_REVIEW_OPS, - AssistantAction, -) +from src.services.llm_provider import LLMProviderService + +from ._dataset_review_dispatch import _dispatch_dataset_review_intent +from ._history import _coerce_query_bool +from ._llm_planner import _check_any_permission from ._resolvers import ( _build_task_observability_summary, _extract_result_deep_links, @@ -33,9 +33,10 @@ from ._resolvers import ( _resolve_env_id, _resolve_provider_id, ) -from ._history import _coerce_query_bool -from ._llm_planner import _check_any_permission -from ._dataset_review_dispatch import _dispatch_dataset_review_intent +from ._schemas import ( + _DATASET_REVIEW_OPS, + AssistantAction, +) git_service = GitService() @@ -45,10 +46,10 @@ git_service = GitService() # @PRE: state was classified as needs_clarification for current intent/error combination. # @POST: Returned text is human-readable and actionable for target operation. def _clarification_text_for_intent( - intent: Optional[Dict[str, Any]], detail_text: str + intent: dict[str, Any] | None, detail_text: str ) -> str: operation = (intent or {}).get("operation") - guidance_by_operation: Dict[str, str] = { + guidance_by_operation: dict[str, str] = { "run_llm_validation": ( "Нужно уточнение для запуска LLM-валидации: Укажите дашборд (id или slug), окружение и провайдер LLM." ), @@ -72,12 +73,12 @@ def _clarification_text_for_intent( # @PRE: actions is a non-empty list of planned review actions. # @POST: Returns a formatted summary string suitable for display to the user. # @SIDE_EFFECT: None - pure formatting function. -async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: ConfigManager, db: Session) -> str: +async def _async_confirmation_summary(intent: dict[str, Any], config_manager: ConfigManager, db: Session) -> str: with belief_scope('_confirmation_summary'): logger.reason('Belief protocol reasoning checkpoint for _confirmation_summary') operation = intent.get('operation', '') entities = intent.get('entities', {}) - descriptions: Dict[str, str] = {'create_branch': 'создание ветки{branch} для дашборда{dashboard}', 'commit_changes': 'коммит изменений для дашборда{dashboard}', 'deploy_dashboard': 'деплой дашборда{dashboard} в окружение{env}', 'execute_migration': 'миграция дашборда{dashboard} с{src} на{tgt}', 'run_backup': 'бэкап окружения{env}{dashboard}', 'run_llm_validation': 'LLM-валидация дашборда{dashboard}{env}', 'run_llm_documentation': 'генерация документации для датасета{dataset}{env}'} + descriptions: dict[str, str] = {'create_branch': 'создание ветки{branch} для дашборда{dashboard}', 'commit_changes': 'коммит изменений для дашборда{dashboard}', 'deploy_dashboard': 'деплой дашборда{dashboard} в окружение{env}', 'execute_migration': 'миграция дашборда{dashboard} с{src} на{tgt}', 'run_backup': 'бэкап окружения{env}{dashboard}', 'run_llm_validation': 'LLM-валидация дашборда{dashboard}{env}', 'run_llm_documentation': 'генерация документации для датасета{dataset}{env}'} template = descriptions.get(operation) if not template: logger.reflect('Belief protocol postcondition checkpoint for _confirmation_summary') @@ -98,8 +99,8 @@ async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: Co if dry_run_enabled: try: from src.core.migration.dry_run_orchestrator import MigrationDryRunService - from src.models.dashboard import DashboardSelection from src.core.superset_client import SupersetClient + from src.models.dashboard import DashboardSelection src_token = entities.get('source_env') tgt_token = entities.get('target_env') dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=src_token) @@ -148,7 +149,7 @@ async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: Co # @PRE: intent operation is known and actor permissions are validated per operation. # @POST: Returns response text, optional task id, and UI actions for follow-up. # @INVARIANT: unsupported operations are rejected via HTTPException(400). -async def _dispatch_intent(intent: Dict[str, Any], current_user: User, task_manager: TaskManager, config_manager: ConfigManager, db: Session) -> Tuple[str, Optional[str], List[AssistantAction]]: +async def _dispatch_intent(intent: dict[str, Any], current_user: User, task_manager: TaskManager, config_manager: ConfigManager, db: Session) -> tuple[str, str | None, list[AssistantAction]]: with belief_scope('_dispatch_intent'): logger.reason('Belief protocol reasoning checkpoint for _dispatch_intent') operation = intent.get('operation') @@ -164,7 +165,7 @@ async def _dispatch_intent(intent: Dict[str, Any], current_user: User, task_mana if not available: logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') return ('Сейчас нет доступных для вас операций ассистента.', None, []) - commands = '\n'.join((f'- {item}' for item in available)) + commands = '\n'.join(f'- {item}' for item in available) text = f'Вот что я могу сделать для вас:\n{commands}\n\nПример: `запусти миграцию с dev на prod для дашборда 42`.' logger.reflect('Belief protocol postcondition checkpoint for _dispatch_intent') return (text, None, []) @@ -248,7 +249,7 @@ async def _dispatch_intent(intent: Dict[str, Any], current_user: User, task_mana raise HTTPException(status_code=422, detail='Missing source_env/target_env') if not dashboard_id and (not dashboard_ref): raise HTTPException(status_code=422, detail='Missing dashboard_id/dashboard_ref') - migration_params: Dict[str, Any] = {'source_env_id': src, 'target_env_id': tgt, 'replace_db_config': _coerce_query_bool(entities.get('replace_db_config', False)), 'fix_cross_filters': _coerce_query_bool(entities.get('fix_cross_filters', True))} + migration_params: dict[str, Any] = {'source_env_id': src, 'target_env_id': tgt, 'replace_db_config': _coerce_query_bool(entities.get('replace_db_config', False)), 'fix_cross_filters': _coerce_query_bool(entities.get('fix_cross_filters', True))} if dashboard_id: migration_params['selected_ids'] = [dashboard_id] else: @@ -262,7 +263,7 @@ async def _dispatch_intent(intent: Dict[str, Any], current_user: User, task_mana env_id = _resolve_env_id(env_token, config_manager) if not env_id: raise HTTPException(status_code=400, detail='Missing or unknown environment') - params: Dict[str, Any] = {'environment_id': env_id} + params: dict[str, Any] = {'environment_id': env_id} if entities.get('dashboard_id') or entities.get('dashboard_ref'): dashboard_id = _resolve_dashboard_id_entity(entities, config_manager, env_hint=env_token) if not dashboard_id: diff --git a/backend/src/api/routes/assistant/_history.py b/backend/src/api/routes/assistant/_history.py index f2f4a6f0..1be61451 100644 --- a/backend/src/api/routes/assistant/_history.py +++ b/backend/src/api/routes/assistant/_history.py @@ -8,21 +8,21 @@ from __future__ import annotations import uuid from datetime import datetime, timedelta -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger +from src.core.logger import logger from src.models.assistant import ( AssistantAuditRecord, AssistantConfirmationRecord, AssistantMessageRecord, ) + from ._schemas import ( ASSISTANT_ARCHIVE_AFTER_DAYS, - ASSISTANT_MESSAGE_TTL_DAYS, ASSISTANT_AUDIT, - CONFIRMATIONS, + ASSISTANT_MESSAGE_TTL_DAYS, CONVERSATIONS, ConfirmationRecord, ) @@ -43,9 +43,9 @@ def _append_history( conversation_id: str, role: str, text: str, - state: Optional[str] = None, - task_id: Optional[str] = None, - confirmation_id: Optional[str] = None, + state: str | None = None, + task_id: str | None = None, + confirmation_id: str | None = None, ): key = (user_id, conversation_id) if key not in CONVERSATIONS: @@ -81,10 +81,10 @@ def _persist_message( conversation_id: str, role: str, text: str, - state: Optional[str] = None, - task_id: Optional[str] = None, - confirmation_id: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, + state: str | None = None, + task_id: str | None = None, + confirmation_id: str | None = None, + metadata: dict[str, Any] | None = None, ): try: row = AssistantMessageRecord( @@ -116,7 +116,7 @@ def _persist_message( # @PRE: payload describes decision/outcome fields. # @POST: ASSISTANT_AUDIT list for user contains new timestamped entry. # @INVARIANT: persisted in-memory audit entry always contains created_at in ISO format. -def _audit(user_id: str, payload: Dict[str, Any]): +def _audit(user_id: str, payload: dict[str, Any]): if user_id not in ASSISTANT_AUDIT: ASSISTANT_AUDIT[user_id] = [] ASSISTANT_AUDIT[user_id].append( @@ -133,7 +133,7 @@ def _audit(user_id: str, payload: Dict[str, Any]): # @PRE: db session is writable and payload is JSON-serializable. # @POST: Audit row is committed or failure is logged with rollback. def _persist_audit( - db: Session, user_id: str, payload: Dict[str, Any], conversation_id: Optional[str] + db: Session, user_id: str, payload: dict[str, Any], conversation_id: str | None ): try: row = AssistantAuditRecord( @@ -213,7 +213,7 @@ def _update_confirmation_state(db: Session, confirmation_id: str, state: str): # @POST: Returns ConfirmationRecord when found, otherwise None. def _load_confirmation_from_db( db: Session, confirmation_id: str -) -> Optional[ConfirmationRecord]: +) -> ConfirmationRecord | None: row = ( db.query(AssistantConfirmationRecord) .filter(AssistantConfirmationRecord.id == confirmation_id) @@ -240,7 +240,7 @@ def _load_confirmation_from_db( # @BRIEF Resolve active conversation id in memory or create a new one. # @PRE: user_id identifies current actor. # @POST: Returns stable conversation id and updates USER_ACTIVE_CONVERSATION. -def _ensure_conversation(user_id: str, conversation_id: Optional[str]) -> str: +def _ensure_conversation(user_id: str, conversation_id: str | None) -> str: if conversation_id: from ._schemas import USER_ACTIVE_CONVERSATION USER_ACTIVE_CONVERSATION[user_id] = conversation_id @@ -264,7 +264,7 @@ def _ensure_conversation(user_id: str, conversation_id: Optional[str]) -> str: # @PRE: user_id and db session are available. # @POST: Returns conversation id and updates USER_ACTIVE_CONVERSATION cache. def _resolve_or_create_conversation( - user_id: str, conversation_id: Optional[str], db: Session + user_id: str, conversation_id: str | None, db: Session ) -> str: from ._schemas import USER_ACTIVE_CONVERSATION @@ -316,7 +316,7 @@ def _cleanup_history_ttl(db: Session, user_id: str): f"[assistant.history][ttl_cleanup_failed] user={user_id} error={exc}" ) - stale_keys: List[Tuple[str, str]] = [] + stale_keys: list[tuple[str, str]] = [] for key, items in CONVERSATIONS.items(): if key[0] != user_id: continue @@ -341,7 +341,7 @@ def _cleanup_history_ttl(db: Session, user_id: str): # @BRIEF Determine archived state for a conversation based on last update timestamp. # @PRE: updated_at can be null for empty conversations. # @POST: Returns True when conversation inactivity exceeds archive threshold. -def _is_conversation_archived(updated_at: Optional[datetime]) -> bool: +def _is_conversation_archived(updated_at: datetime | None) -> bool: if not updated_at: return False cutoff = datetime.utcnow() - timedelta(days=ASSISTANT_ARCHIVE_AFTER_DAYS) diff --git a/backend/src/api/routes/assistant/_llm_planner.py b/backend/src/api/routes/assistant/_llm_planner.py index ac68fe05..b065c368 100644 --- a/backend/src/api/routes/assistant/_llm_planner.py +++ b/backend/src/api/routes/assistant/_llm_planner.py @@ -8,32 +8,31 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from fastapi import HTTPException from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger from src.core.config_manager import ConfigManager from src.dependencies import has_permission from src.schemas.auth import User -from src.services.llm_provider import LLMProviderService from src.services.llm_prompt_templates import resolve_bound_provider_id -from ._schemas import ( - INTENT_PERMISSION_CHECKS, - AssistantAction, -) +from src.services.llm_provider import LLMProviderService + from ._resolvers import ( _get_default_environment_id, ) +from ._schemas import ( + INTENT_PERMISSION_CHECKS, +) # #region _check_any_permission [C:2] [TYPE Function] # @BRIEF Validate user against alternative permission checks (logical OR). # @PRE: checks list contains resource-action tuples. # @POST: Returns on first successful permission; raises 403-like HTTPException otherwise. -def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): - errors: List[HTTPException] = [] +def _check_any_permission(current_user: User, checks: list[tuple[str, str]]): + errors: list[HTTPException] = [] for resource, action in checks: try: has_permission(resource, action)(current_user) @@ -55,7 +54,7 @@ def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): # @BRIEF Check whether user has at least one permission tuple from the provided list. # @PRE: current_user and checks list are valid. # @POST: Returns True when at least one permission check passes. -def _has_any_permission(current_user: User, checks: List[Tuple[str, str]]) -> bool: +def _has_any_permission(current_user: User, checks: list[tuple[str, str]]) -> bool: try: _check_any_permission(current_user, checks) return True @@ -75,8 +74,8 @@ def _build_tool_catalog( current_user: User, config_manager: ConfigManager, db: Session, - dataset_review_context: Optional[Dict[str, Any]] = None, -) -> List[Dict[str, Any]]: + dataset_review_context: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: envs = config_manager.get_environments() default_env_id = _get_default_environment_id(config_manager) providers = LLMProviderService(db).get_all_providers() @@ -95,7 +94,7 @@ def _build_tool_catalog( resolve_bound_provider_id(llm_settings, "documentation") or fallback_provider ) - candidates: List[Dict[str, Any]] = [ + candidates: list[dict[str, Any]] = [ { "operation": "show_capabilities", "domain": "assistant", @@ -201,7 +200,7 @@ def _build_tool_catalog( }, ] - available: List[Dict[str, Any]] = [] + available: list[dict[str, Any]] = [] for tool in candidates: checks = INTENT_PERMISSION_CHECKS.get(tool["operation"], []) if checks and not _has_any_permission(current_user, checks): @@ -209,7 +208,7 @@ def _build_tool_catalog( available.append(tool) if dataset_review_context is not None: - dataset_tools: List[Dict[str, Any]] = [ + dataset_tools: list[dict[str, Any]] = [ { "operation": "dataset_review_answer_context", "domain": "dataset_review", @@ -272,7 +271,7 @@ def _build_tool_catalog( # @BRIEF Normalize intent entity value types from LLM output to route-compatible values. # @PRE: intent contains entities dict or missing entities. # @POST: Returned intent has numeric ids coerced where possible and string values stripped. -def _coerce_intent_entities(intent: Dict[str, Any]) -> Dict[str, Any]: +def _coerce_intent_entities(intent: dict[str, Any]) -> dict[str, Any]: entities = intent.get("entities") if not isinstance(entities, dict): intent["entities"] = {} diff --git a/backend/src/api/routes/assistant/_llm_planner_intent.py b/backend/src/api/routes/assistant/_llm_planner_intent.py index c06edb2e..a4724b8a 100644 --- a/backend/src/api/routes/assistant/_llm_planner_intent.py +++ b/backend/src/api/routes/assistant/_llm_planner_intent.py @@ -8,29 +8,29 @@ from __future__ import annotations import json -from typing import Any, Dict, List, Optional +from typing import Any from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger from src.core.config_manager import ConfigManager +from src.core.logger import logger +from src.plugins.llm_analysis.models import LLMProviderType +from src.plugins.llm_analysis.service import LLMClient from src.schemas.auth import User -from src.services.llm_provider import LLMProviderService from src.services.llm_prompt_templates import ( normalize_llm_settings, - resolve_bound_provider_id, -) -from src.plugins.llm_analysis.service import LLMClient -from src.plugins.llm_analysis.models import LLMProviderType -from ._schemas import INTENT_PERMISSION_CHECKS -from ._resolvers import ( - _is_production_env, - _resolve_provider_id, ) +from src.services.llm_provider import LLMProviderService + from ._llm_planner import ( _check_any_permission, _coerce_intent_entities, ) +from ._resolvers import ( + _is_production_env, + _resolve_provider_id, +) +from ._schemas import INTENT_PERMISSION_CHECKS # #region _plan_intent_with_llm [C:2] [TYPE Function] @@ -39,10 +39,10 @@ from ._llm_planner import ( # @POST: Returns normalized intent dict when planning succeeds; otherwise None. async def _plan_intent_with_llm( message: str, - tools: List[Dict[str, Any]], + tools: list[dict[str, Any]], db: Session, config_manager: ConfigManager, -) -> Optional[Dict[str, Any]]: +) -> dict[str, Any] | None: if not tools: return None @@ -158,7 +158,7 @@ async def _plan_intent_with_llm( # @BRIEF Validate user permissions for parsed intent before confirmation/dispatch. # @PRE: intent.operation is present for known assistant command domains. # @POST: Returns if authorized; raises HTTPException(403) when denied. -def _authorize_intent(intent: Dict[str, Any], current_user: User): +def _authorize_intent(intent: dict[str, Any], current_user: User): operation = intent.get("operation") if operation in INTENT_PERMISSION_CHECKS: _check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation]) diff --git a/backend/src/api/routes/assistant/_resolvers.py b/backend/src/api/routes/assistant/_resolvers.py index c340274b..67374806 100644 --- a/backend/src/api/routes/assistant/_resolvers.py +++ b/backend/src/api/routes/assistant/_resolvers.py @@ -8,16 +8,15 @@ from __future__ import annotations import re -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, cast from sqlalchemy.orm import Session -from src.core.logger import belief_scope, logger from src.core.config_manager import ConfigManager +from src.core.logger import logger from src.core.superset_client import SupersetClient -from src.services.llm_provider import LLMProviderService from src.services.llm_prompt_templates import resolve_bound_provider_id -from src.schemas.auth import User +from src.services.llm_provider import LLMProviderService logger = cast(Any, logger) @@ -26,7 +25,7 @@ logger = cast(Any, logger) # @BRIEF Extract first regex match group from text by ordered pattern list. # @PRE: patterns contain at least one capture group. # @POST: Returns first matched token or None. -def _extract_id(text: str, patterns: List[str]) -> Optional[str]: +def _extract_id(text: str, patterns: list[str]) -> str | None: for p in patterns: m = re.search(p, text, flags=re.IGNORECASE) if m: @@ -41,8 +40,8 @@ def _extract_id(text: str, patterns: List[str]) -> Optional[str]: # @PRE: config_manager provides environment list. # @POST: Returns matched environment id or None. def _resolve_env_id( - token: Optional[str], config_manager: ConfigManager -) -> Optional[str]: + token: str | None, config_manager: ConfigManager +) -> str | None: if not token: return None @@ -60,7 +59,7 @@ def _resolve_env_id( # @BRIEF Determine whether environment token resolves to production-like target. # @PRE: config_manager provides environments or token text is provided. # @POST: Returns True for production/prod synonyms, else False. -def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> bool: +def _is_production_env(token: str | None, config_manager: ConfigManager) -> bool: env_id = _resolve_env_id(token, config_manager) if not env_id: return (token or "").strip().lower() in {"prod", "production", "прод"} @@ -79,11 +78,11 @@ def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> b # @PRE: db session can load provider list through LLMProviderService. # @POST: Returns provider id or None when no providers configured. def _resolve_provider_id( - provider_token: Optional[str], + provider_token: str | None, db: Session, - config_manager: Optional[ConfigManager] = None, - task_key: Optional[str] = None, -) -> Optional[str]: + config_manager: ConfigManager | None = None, + task_key: str | None = None, +) -> str | None: service = LLMProviderService(db) providers = service.get_all_providers() if not providers: @@ -114,7 +113,7 @@ def _resolve_provider_id( # @BRIEF Resolve default environment id from settings or first configured environment. # @PRE: config_manager returns environments list. # @POST: Returns default environment id or None when environment list is empty. -def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: +def _get_default_environment_id(config_manager: ConfigManager) -> str | None: configured = config_manager.get_environments() if not configured: return None @@ -139,10 +138,10 @@ def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: # @PRE: dashboard_ref is a non-empty string-like token. # @POST: Returns dashboard id when uniquely matched, otherwise None. def _resolve_dashboard_id_by_ref( - dashboard_ref: Optional[str], - env_id: Optional[str], + dashboard_ref: str | None, + env_id: str | None, config_manager: ConfigManager, -) -> Optional[int]: +) -> int | None: if not dashboard_ref or not env_id: return None env = next( @@ -191,10 +190,10 @@ def _resolve_dashboard_id_by_ref( # @PRE: entities may contain dashboard_id as int/str and optional dashboard_ref. # @POST: Returns resolved dashboard id or None when ambiguous/unresolvable. def _resolve_dashboard_id_entity( - entities: Dict[str, Any], + entities: dict[str, Any], config_manager: ConfigManager, - env_hint: Optional[str] = None, -) -> Optional[int]: + env_hint: str | None = None, +) -> int | None: raw_dashboard_id = entities.get("dashboard_id") dashboard_ref = entities.get("dashboard_ref") @@ -232,7 +231,7 @@ def _resolve_dashboard_id_entity( # @PRE: environment id may be None. # @POST: Returns matching environment name or fallback id. def _get_environment_name_by_id( - env_id: Optional[str], config_manager: ConfigManager + env_id: str | None, config_manager: ConfigManager ) -> str: if not env_id: return "unknown" @@ -250,15 +249,15 @@ def _get_environment_name_by_id( # @POST: Returns zero or more assistant actions for dashboard open/diff. def _extract_result_deep_links( task: Any, config_manager: ConfigManager -) -> List: +) -> list: from ._schemas import AssistantAction plugin_id = getattr(task, "plugin_id", None) params = getattr(task, "params", {}) or {} result = getattr(task, "result", {}) or {} actions: list = [] - dashboard_id: Optional[int] = None - env_id: Optional[str] = None + dashboard_id: int | None = None + env_id: str | None = None if plugin_id == "superset-migration": migrated = ( diff --git a/backend/src/api/routes/assistant/_routes.py b/backend/src/api/routes/assistant/_routes.py index e00727b9..0d0baed0 100644 --- a/backend/src/api/routes/assistant/_routes.py +++ b/backend/src/api/routes/assistant/_routes.py @@ -14,54 +14,22 @@ from __future__ import annotations import uuid from datetime import datetime -from typing import Any, Dict, Optional -from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session -from sqlalchemy import desc +from src.core.config_manager import ConfigManager +from src.core.database import get_db from src.core.logger import belief_scope, logger from src.core.task_manager import TaskManager from src.dependencies import ( + get_config_manager, get_current_user, get_task_manager, - get_config_manager, - has_permission, -) -from src.core.config_manager import ConfigManager -from src.core.database import get_db -from src.models.assistant import ( - AssistantAuditRecord, - AssistantMessageRecord, ) from src.schemas.auth import User -from ._schemas import ( - _SAFE_OPS, - ASSISTANT_AUDIT, - CONFIRMATIONS, - CONVERSATIONS, - USER_ACTIVE_CONVERSATION, - AssistantAction, - AssistantMessageRequest, - AssistantMessageResponse, - ConfirmationRecord, -) -from ._history import ( - _append_history, - _audit, - _cleanup_history_ttl, - _coerce_query_bool, - _is_conversation_archived, - _load_confirmation_from_db, - _persist_audit, - _persist_confirmation, - _persist_message, - _resolve_or_create_conversation, - _update_confirmation_state, -) + from ._command_parser import _parse_command -from ._llm_planner import _build_tool_catalog -from ._llm_planner_intent import _authorize_intent, _plan_intent_with_llm from ._dataset_review import ( _load_dataset_review_context, _plan_dataset_review_intent, @@ -71,6 +39,26 @@ from ._dispatch import ( _clarification_text_for_intent, _dispatch_intent, ) +from ._history import ( + _append_history, + _audit, + _load_confirmation_from_db, + _persist_audit, + _persist_confirmation, + _persist_message, + _resolve_or_create_conversation, + _update_confirmation_state, +) +from ._llm_planner import _build_tool_catalog +from ._llm_planner_intent import _authorize_intent, _plan_intent_with_llm +from ._schemas import ( + _SAFE_OPS, + CONFIRMATIONS, + AssistantAction, + AssistantMessageRequest, + AssistantMessageResponse, + ConfirmationRecord, +) router = APIRouter(tags=["Assistant"]) diff --git a/backend/src/api/routes/assistant/_schemas.py b/backend/src/api/routes/assistant/_schemas.py index f1a81618..928874a6 100644 --- a/backend/src/api/routes/assistant/_schemas.py +++ b/backend/src/api/routes/assistant/_schemas.py @@ -8,12 +8,10 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from pydantic import BaseModel, Field -from src.schemas.auth import User - # #region AssistantMessageRequest [C:1] [TYPE Class] # @BRIEF Input payload for assistant message endpoint. @@ -24,9 +22,9 @@ from src.schemas.auth import User # @POST: Request object provides message text and optional conversation binding. # @INVARIANT: message is always non-empty and no longer than 4000 characters. class AssistantMessageRequest(BaseModel): - conversation_id: Optional[str] = None + conversation_id: str | None = None message: str = Field(..., min_length=1, max_length=4000) - dataset_review_session_id: Optional[str] = None + dataset_review_session_id: str | None = None # #endregion AssistantMessageRequest @@ -43,7 +41,7 @@ class AssistantMessageRequest(BaseModel): class AssistantAction(BaseModel): type: str label: str - target: Optional[str] = None + target: str | None = None # #endregion AssistantAction @@ -64,10 +62,10 @@ class AssistantMessageResponse(BaseModel): response_id: str state: str text: str - intent: Optional[Dict[str, Any]] = None - confirmation_id: Optional[str] = None - task_id: Optional[str] = None - actions: List[AssistantAction] = Field(default_factory=list) + intent: dict[str, Any] | None = None + confirmation_id: str | None = None + task_id: str | None = None + actions: list[AssistantAction] = Field(default_factory=list) created_at: datetime @@ -88,8 +86,8 @@ class ConfirmationRecord(BaseModel): id: str user_id: str conversation_id: str - intent: Dict[str, Any] - dispatch: Dict[str, Any] + intent: dict[str, Any] + dispatch: dict[str, Any] expires_at: datetime state: str = "pending" created_at: datetime @@ -100,10 +98,10 @@ class ConfirmationRecord(BaseModel): # --- In-memory stores --- -CONVERSATIONS: Dict[Tuple[str, str], List[Dict[str, Any]]] = {} -USER_ACTIVE_CONVERSATION: Dict[str, str] = {} -CONFIRMATIONS: Dict[str, ConfirmationRecord] = {} -ASSISTANT_AUDIT: Dict[str, List[Dict[str, Any]]] = {} +CONVERSATIONS: dict[tuple[str, str], list[dict[str, Any]]] = {} +USER_ACTIVE_CONVERSATION: dict[str, str] = {} +CONFIRMATIONS: dict[str, ConfirmationRecord] = {} +ASSISTANT_AUDIT: dict[str, list[dict[str, Any]]] = {} ASSISTANT_ARCHIVE_AFTER_DAYS = 14 ASSISTANT_MESSAGE_TTL_DAYS = 90 @@ -121,7 +119,7 @@ _DATASET_REVIEW_OPS = { "dataset_review_generate_sql_preview", } -INTENT_PERMISSION_CHECKS: Dict[str, List[Tuple[str, str]]] = { +INTENT_PERMISSION_CHECKS: dict[str, list[tuple[str, str]]] = { "get_task_status": [("tasks", "READ")], "create_branch": [("plugin:git", "EXECUTE")], "commit_changes": [("plugin:git", "EXECUTE")], diff --git a/backend/src/api/routes/clean_release.py b/backend/src/api/routes/clean_release.py index 8c42df36..bfc241dc 100644 --- a/backend/src/api/routes/clean_release.py +++ b/backend/src/api/routes/clean_release.py @@ -10,45 +10,38 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import Any, Dict, List +from datetime import UTC, datetime +from typing import Any from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, Field from ...core.logger import belief_scope, logger -from ...dependencies import get_clean_release_repository, get_config_manager -from ...services.clean_release.preparation_service import prepare_candidate -from ...services.clean_release.repository import CleanReleaseRepository +from ...dependencies import get_clean_release_repository +from ...models.clean_release import ( + CandidateArtifact, + ComplianceStageRun, + ComplianceViolation, + ReleaseCandidate, +) from ...services.clean_release.compliance_orchestrator import ( CleanComplianceOrchestrator, ) -from ...services.clean_release.report_builder import ComplianceReportBuilder -from ...services.clean_release.compliance_execution_service import ( - ComplianceExecutionService, - ComplianceRunError, -) from ...services.clean_release.dto import ( CandidateDTO, - ManifestDTO, CandidateOverviewDTO, - ComplianceRunDTO, + ManifestDTO, ) from ...services.clean_release.enums import ( + CandidateStatus, ComplianceDecision, ComplianceStageName, - ViolationCategory, - ViolationSeverity, RunStatus, - CandidateStatus, -) -from ...models.clean_release import ( - ComplianceRun, - ComplianceStageRun, - ComplianceViolation, - CandidateArtifact, - ReleaseCandidate, + ViolationSeverity, ) +from ...services.clean_release.preparation_service import prepare_candidate +from ...services.clean_release.report_builder import ComplianceReportBuilder +from ...services.clean_release.repository import CleanReleaseRepository router = APIRouter(prefix="/api/clean-release", tags=["Clean Release"]) @@ -57,8 +50,8 @@ router = APIRouter(prefix="/api/clean-release", tags=["Clean Release"]) # @BRIEF Request schema for candidate preparation endpoint. class PrepareCandidateRequest(BaseModel): candidate_id: str = Field(min_length=1) - artifacts: List[Dict[str, Any]] = Field(default_factory=list) - sources: List[str] = Field(default_factory=list) + artifacts: list[dict[str, Any]] = Field(default_factory=list) + sources: list[str] = Field(default_factory=list) operator_id: str = Field(min_length=1) @@ -92,7 +85,7 @@ class RegisterCandidateRequest(BaseModel): # #region ImportArtifactsRequest [TYPE Class] # @BRIEF Request schema for candidate artifact import endpoint. class ImportArtifactsRequest(BaseModel): - artifacts: List[Dict[str, Any]] = Field(default_factory=list) + artifacts: list[dict[str, Any]] = Field(default_factory=list) # #endregion ImportArtifactsRequest @@ -140,7 +133,7 @@ async def register_candidate_v2_endpoint( version=payload.version, source_snapshot_ref=payload.source_snapshot_ref, created_by=payload.created_by, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.DRAFT.value, ) repository.save_candidate(candidate) @@ -292,7 +285,7 @@ async def get_candidate_overview_v2_endpoint( sorted( runs, key=lambda run: run.requested_at - or datetime.min.replace(tzinfo=timezone.utc), + or datetime.min.replace(tzinfo=UTC), reverse=True, )[0] if runs @@ -317,7 +310,7 @@ async def get_candidate_overview_v2_endpoint( sorted( [item for item in approval_decisions if item.candidate_id == candidate_id], key=lambda item: item.decided_at - or datetime.min.replace(tzinfo=timezone.utc), + or datetime.min.replace(tzinfo=UTC), reverse=True, )[0] if approval_decisions @@ -330,7 +323,7 @@ async def get_candidate_overview_v2_endpoint( sorted( [item for item in publication_records if item.candidate_id == candidate_id], key=lambda item: item.published_at - or datetime.min.replace(tzinfo=timezone.utc), + or datetime.min.replace(tzinfo=UTC), reverse=True, )[0] if publication_records diff --git a/backend/src/api/routes/clean_release_v2.py b/backend/src/api/routes/clean_release_v2.py index 562d517b..9a9a7feb 100644 --- a/backend/src/api/routes/clean_release_v2.py +++ b/backend/src/api/routes/clean_release_v2.py @@ -8,26 +8,28 @@ # @POST: Candidate registration, approval, publication, and revocation routes are registered without behavior changes. # @SIDE_EFFECT: Persists candidate lifecycle state through clean release services and repository adapters. +from datetime import UTC, datetime +from typing import Any + from fastapi import APIRouter, Depends, HTTPException, status -from typing import List, Dict, Any -from datetime import datetime, timezone + +from ...dependencies import get_clean_release_repository +from ...models.clean_release import ( + CandidateArtifact, + DistributionManifest, + ReleaseCandidate, +) from ...services.clean_release.approval_service import ( approve_candidate, reject_candidate, ) +from ...services.clean_release.dto import CandidateDTO, ManifestDTO +from ...services.clean_release.enums import CandidateStatus from ...services.clean_release.publication_service import ( publish_candidate, revoke_publication, ) from ...services.clean_release.repository import CleanReleaseRepository -from ...dependencies import get_clean_release_repository -from ...services.clean_release.enums import CandidateStatus -from ...models.clean_release import ( - ReleaseCandidate, - CandidateArtifact, - DistributionManifest, -) -from ...services.clean_release.dto import CandidateDTO, ManifestDTO router = APIRouter(prefix="/api/v2/clean-release", tags=["Clean Release V2"]) @@ -69,7 +71,7 @@ class RevokeRequest(dict): "/candidates", response_model=CandidateDTO, status_code=status.HTTP_201_CREATED ) async def register_candidate( - payload: Dict[str, Any], + payload: dict[str, Any], repository: CleanReleaseRepository = Depends(get_clean_release_repository), ): candidate = ReleaseCandidate( @@ -77,7 +79,7 @@ async def register_candidate( version=payload["version"], source_snapshot_ref=payload["source_snapshot_ref"], created_by=payload["created_by"], - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.DRAFT.value, ) repository.save_candidate(candidate) @@ -102,7 +104,7 @@ async def register_candidate( @router.post("/candidates/{candidate_id}/artifacts") async def import_artifacts( candidate_id: str, - payload: Dict[str, Any], + payload: dict[str, Any], repository: CleanReleaseRepository = Depends(get_clean_release_repository), ): candidate = repository.get_candidate(candidate_id) @@ -152,7 +154,7 @@ async def build_manifest( manifest_digest="hash-123", artifacts_digest="art-hash-123", created_by="system", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), source_snapshot_ref=candidate.source_snapshot_ref, content_json={"items": [], "summary": {}}, ) @@ -180,7 +182,7 @@ async def build_manifest( @router.post("/candidates/{candidate_id}/approve") async def approve_candidate_endpoint( candidate_id: str, - payload: Dict[str, Any], + payload: dict[str, Any], repository: CleanReleaseRepository = Depends(get_clean_release_repository), ): try: @@ -191,7 +193,7 @@ async def approve_candidate_endpoint( decided_by=str(payload["decided_by"]), comment=payload.get("comment"), ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: raise HTTPException( status_code=409, detail={"message": str(exc), "code": "APPROVAL_GATE_ERROR"} ) @@ -208,7 +210,7 @@ async def approve_candidate_endpoint( @router.post("/candidates/{candidate_id}/reject") async def reject_candidate_endpoint( candidate_id: str, - payload: Dict[str, Any], + payload: dict[str, Any], repository: CleanReleaseRepository = Depends(get_clean_release_repository), ): try: @@ -219,7 +221,7 @@ async def reject_candidate_endpoint( decided_by=str(payload["decided_by"]), comment=payload.get("comment"), ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: raise HTTPException( status_code=409, detail={"message": str(exc), "code": "APPROVAL_GATE_ERROR"} ) @@ -236,7 +238,7 @@ async def reject_candidate_endpoint( @router.post("/candidates/{candidate_id}/publish") async def publish_candidate_endpoint( candidate_id: str, - payload: Dict[str, Any], + payload: dict[str, Any], repository: CleanReleaseRepository = Depends(get_clean_release_repository), ): try: @@ -248,7 +250,7 @@ async def publish_candidate_endpoint( target_channel=str(payload["target_channel"]), publication_ref=payload.get("publication_ref"), ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: raise HTTPException( status_code=409, detail={"message": str(exc), "code": "PUBLICATION_GATE_ERROR"}, @@ -280,7 +282,7 @@ async def publish_candidate_endpoint( @router.post("/publications/{publication_id}/revoke") async def revoke_publication_endpoint( publication_id: str, - payload: Dict[str, Any], + payload: dict[str, Any], repository: CleanReleaseRepository = Depends(get_clean_release_repository), ): try: @@ -290,7 +292,7 @@ async def revoke_publication_endpoint( revoked_by=str(payload["revoked_by"]), comment=payload.get("comment"), ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: raise HTTPException( status_code=409, detail={"message": str(exc), "code": "PUBLICATION_GATE_ERROR"}, diff --git a/backend/src/api/routes/connections.py b/backend/src/api/routes/connections.py index a748dff3..4be63c50 100644 --- a/backend/src/api/routes/connections.py +++ b/backend/src/api/routes/connections.py @@ -4,14 +4,15 @@ # @RELATION DEPENDS_ON -> [get_db] # @RELATION DEPENDS_ON -> [ConnectionConfig] -from typing import List, Optional -from fastapi import APIRouter, Depends, HTTPException, status -from sqlalchemy.orm import Session -from ...core.database import get_db, ensure_connection_configs_table -from ...models.connection import ConnectionConfig -from pydantic import BaseModel from datetime import datetime -from ...core.logger import logger, belief_scope + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from ...core.database import ensure_connection_configs_table, get_db +from ...core.logger import belief_scope, logger +from ...models.connection import ConnectionConfig router = APIRouter() @@ -36,10 +37,10 @@ class ConnectionSchema(BaseModel): id: str name: str type: str - host: Optional[str] = None - port: Optional[int] = None - database: Optional[str] = None - username: Optional[str] = None + host: str | None = None + port: int | None = None + database: str | None = None + username: str | None = None created_at: datetime class Config: @@ -55,11 +56,11 @@ class ConnectionSchema(BaseModel): class ConnectionCreate(BaseModel): name: str type: str - host: Optional[str] = None - port: Optional[int] = None - database: Optional[str] = None - username: Optional[str] = None - password: Optional[str] = None + host: str | None = None + port: int | None = None + database: str | None = None + username: str | None = None + password: str | None = None # #endregion ConnectionCreate @@ -71,7 +72,7 @@ class ConnectionCreate(BaseModel): # @POST: Returns list of connection configs. # @RELATION CALLS -> _ensure_connections_schema # @RELATION DEPENDS_ON -> ConnectionConfig -@router.get("", response_model=List[ConnectionSchema]) +@router.get("", response_model=list[ConnectionSchema]) async def list_connections(db: Session = Depends(get_db)): with belief_scope("ConnectionsRouter.list_connections"): _ensure_connections_schema(db) diff --git a/backend/src/api/routes/dashboards/__init__.py b/backend/src/api/routes/dashboards/__init__.py index 7c94e6f1..e6143e08 100644 --- a/backend/src/api/routes/dashboards/__init__.py +++ b/backend/src/api/routes/dashboards/__init__.py @@ -33,10 +33,10 @@ # # @TEST_INVARIANT: metadata_consistency -> verifies: [dashboard_list_happy, empty_dashboards] +from ._action_routes import * +from ._detail_routes import * +from ._listing_routes import * from ._router import router from ._schemas import * -from ._listing_routes import * -from ._detail_routes import * -from ._action_routes import * # #endregion DashboardsApi diff --git a/backend/src/api/routes/dashboards/_action_routes.py b/backend/src/api/routes/dashboards/_action_routes.py index 66bdcf0c..60c3080b 100644 --- a/backend/src/api/routes/dashboards/_action_routes.py +++ b/backend/src/api/routes/dashboards/_action_routes.py @@ -6,18 +6,19 @@ from fastapi import Depends, HTTPException +from src.core.logger import belief_scope, logger from src.dependencies import ( get_config_manager, get_task_manager, has_permission, ) -from src.core.logger import logger, belief_scope + +from ._router import router from ._schemas import ( - MigrateRequest, BackupRequest, + MigrateRequest, TaskResponse, ) -from ._router import router # #region migrate_dashboards [C:2] [TYPE Function] @@ -91,7 +92,7 @@ async def migrate_dashboards( f"[migrate_dashboards][Coherence:Failed] Failed to create migration task: {e}" ) raise HTTPException( - status_code=503, detail=f"Failed to create migration task: {str(e)}" + status_code=503, detail=f"Failed to create migration task: {e!s}" ) @@ -158,7 +159,7 @@ async def backup_dashboards( f"[backup_dashboards][Coherence:Failed] Failed to create backup task: {e}" ) raise HTTPException( - status_code=503, detail=f"Failed to create backup task: {str(e)}" + status_code=503, detail=f"Failed to create backup task: {e!s}" ) diff --git a/backend/src/api/routes/dashboards/_detail_routes.py b/backend/src/api/routes/dashboards/_detail_routes.py index cd7deaac..553914d1 100644 --- a/backend/src/api/routes/dashboards/_detail_routes.py +++ b/backend/src/api/routes/dashboards/_detail_routes.py @@ -6,23 +6,24 @@ # @RELATION DEPENDS_ON -> [DashboardHelpers] # @RELATION DEPENDS_ON -> [DashboardProjection] -from typing import Optional, List, Dict, Any import re +from typing import Any from urllib.parse import urlparse from fastapi import Depends, HTTPException, Query, Response from fastapi.responses import JSONResponse +from src.core.async_superset_client import AsyncSupersetClient +from src.core.logger import belief_scope, logger +from src.core.superset_client import SupersetClient +from src.core.utils.network import DashboardNotFoundError from src.dependencies import ( get_config_manager, - get_task_manager, get_mapping_service, + get_task_manager, has_permission, ) -from src.core.async_superset_client import AsyncSupersetClient -from src.core.superset_client import SupersetClient -from src.core.logger import logger, belief_scope -from src.core.utils.network import DashboardNotFoundError + from ._helpers import ( _resolve_dashboard_id_from_ref, _resolve_dashboard_id_from_ref_async, @@ -30,6 +31,7 @@ from ._helpers import ( from ._projection import ( _task_matches_dashboard, ) +from ._router import router from ._schemas import ( DashboardDetailResponse, DashboardTaskHistoryItem, @@ -37,7 +39,6 @@ from ._schemas import ( DatabaseMapping, DatabaseMappingsResponse, ) -from ._router import router # #region get_database_mappings [C:2] [TYPE Function] @@ -99,7 +100,7 @@ async def get_database_mappings( f"[get_database_mappings][Coherence:Failed] Failed to get database mappings: {e}" ) raise HTTPException( - status_code=503, detail=f"Failed to get database mappings: {str(e)}" + status_code=503, detail=f"Failed to get database mappings: {e!s}" ) @@ -144,7 +145,7 @@ async def get_dashboard_detail( f"[get_dashboard_detail][Coherence:Failed] Failed to fetch dashboard detail: {e}" ) raise HTTPException( - status_code=503, detail=f"Failed to fetch dashboard detail: {str(e)}" + status_code=503, detail=f"Failed to fetch dashboard detail: {e!s}" ) @@ -158,7 +159,7 @@ async def get_dashboard_detail( @router.get("/{dashboard_ref}/tasks", response_model=DashboardTaskHistoryResponse) async def get_dashboard_tasks_history( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, limit: int = Query(20, ge=1, le=100), config_manager=Depends(get_config_manager), task_manager=Depends(get_task_manager), @@ -168,8 +169,8 @@ async def get_dashboard_tasks_history( "get_dashboard_tasks_history", f"dashboard_ref={dashboard_ref}, env_id={env_id}, limit={limit}", ): - dashboard_id: Optional[int] = None - client: Optional[AsyncSupersetClient] = None + dashboard_id: int | None = None + client: AsyncSupersetClient | None = None try: if dashboard_ref.isdigit(): dashboard_id = int(dashboard_ref) @@ -351,7 +352,7 @@ async def get_dashboard_thumbnail( ) if thumb_response.status_code == 202: - payload_202: Dict[str, Any] = {} + payload_202: dict[str, Any] = {} try: payload_202 = thumb_response.json() except Exception: @@ -373,7 +374,7 @@ async def get_dashboard_thumbnail( ) raise HTTPException( status_code=503, - detail=f"Failed to fetch dashboard thumbnail: {str(e)}", + detail=f"Failed to fetch dashboard thumbnail: {e!s}", ) diff --git a/backend/src/api/routes/dashboards/_helpers.py b/backend/src/api/routes/dashboards/_helpers.py index 292e89fd..79bd6db6 100644 --- a/backend/src/api/routes/dashboards/_helpers.py +++ b/backend/src/api/routes/dashboards/_helpers.py @@ -4,11 +4,12 @@ # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [AsyncSupersetClient] -from typing import List, Optional, Dict, Any +from typing import Any + from fastapi import HTTPException -from src.core.superset_client import SupersetClient + from src.core.async_superset_client import AsyncSupersetClient -from src.core.logger import logger +from src.core.superset_client import SupersetClient # #region _find_dashboard_id_by_slug [C:2] [TYPE Function] @@ -18,7 +19,7 @@ from src.core.logger import logger def _find_dashboard_id_by_slug( client: SupersetClient, dashboard_slug: str, -) -> Optional[int]: +) -> int | None: query_variants = [ { "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], @@ -81,7 +82,7 @@ def _resolve_dashboard_id_from_ref( async def _find_dashboard_id_by_slug_async( client: AsyncSupersetClient, dashboard_slug: str, -) -> Optional[int]: +) -> int | None: query_variants = [ { "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], @@ -140,10 +141,10 @@ async def _resolve_dashboard_id_from_ref_async( # @BRIEF Normalize query filter values to lower-cased non-empty tokens. # @PRE: values may be None or list of strings. # @POST: Returns trimmed normalized list preserving input order. -def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: +def _normalize_filter_values(values: list[str] | None) -> list[str]: if not values: return [] - normalized: List[str] = [] + normalized: list[str] = [] for value in values: token = str(value or "").strip().lower() if token: @@ -158,7 +159,7 @@ def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: # @BRIEF Build comparable git status token for dashboards filtering. # @PRE: dashboard payload may contain git_status or None. # @POST: Returns one of ok|diff|no_repo|error|pending. -def _dashboard_git_filter_value(dashboard: Dict[str, Any]) -> str: +def _dashboard_git_filter_value(dashboard: dict[str, Any]) -> str: git_status = dashboard.get("git_status") or {} sync_status = str(git_status.get("sync_status") or "").strip().upper() has_repo = git_status.get("has_repo") diff --git a/backend/src/api/routes/dashboards/_listing_routes.py b/backend/src/api/routes/dashboards/_listing_routes.py index eab10b38..573cdc43 100644 --- a/backend/src/api/routes/dashboards/_listing_routes.py +++ b/backend/src/api/routes/dashboards/_listing_routes.py @@ -6,25 +6,32 @@ # @RELATION DEPENDS_ON -> [DashboardHelpers] # @RELATION DEPENDS_ON -> [DashboardProjection] -import os -from typing import List, Optional, Dict, Any, Literal +from typing import Any, Literal + from fastapi import Depends, HTTPException, Query from sqlalchemy.orm import Session -from src.dependencies import ( - get_config_manager, get_task_manager, get_resource_service, - get_current_user, has_permission, -) + from src.core.database import get_db -from src.core.logger import logger, belief_scope +from src.core.logger import belief_scope, logger +from src.dependencies import ( + get_config_manager, + get_current_user, + get_resource_service, + get_task_manager, + has_permission, +) from src.models.auth import User from src.services.profile_service import ProfileService -from ._helpers import _normalize_filter_values, _dashboard_git_filter_value + +from ._helpers import _dashboard_git_filter_value, _normalize_filter_values from ._projection import ( - _project_dashboard_response_items, _get_profile_filter_binding, - _resolve_profile_actor_aliases, _matches_dashboard_actor_aliases, + _get_profile_filter_binding, + _matches_dashboard_actor_aliases, + _project_dashboard_response_items, + _resolve_profile_actor_aliases, ) -from ._schemas import EffectiveProfileFilter, DashboardsResponse from ._router import router +from ._schemas import DashboardsResponse, EffectiveProfileFilter # #region get_dashboards [C:3] [TYPE Function] @@ -39,17 +46,17 @@ from ._router import router @router.get("", response_model=DashboardsResponse) async def get_dashboards( env_id: str, - search: Optional[str] = None, + search: str | None = None, page: int = 1, page_size: int = 10, page_context: Literal["dashboards_main", "other"] = Query("dashboards_main"), apply_profile_default: bool = Query(default=True), override_show_all: bool = Query(default=False), - filter_title: Optional[List[str]] = Query(default=None), - filter_git_status: Optional[List[str]] = Query(default=None), - filter_llm_status: Optional[List[str]] = Query(default=None), - filter_changed_on: Optional[List[str]] = Query(default=None), - filter_actor: Optional[List[str]] = Query(default=None), + filter_title: list[str] | None = Query(default=None), + filter_git_status: list[str] | None = Query(default=None), + filter_llm_status: list[str] | None = Query(default=None), + filter_changed_on: list[str] | None = Query(default=None), + filter_actor: list[str] | None = Query(default=None), config_manager=Depends(get_config_manager), task_manager=Depends(get_task_manager), resource_service=Depends(get_resource_service), @@ -75,7 +82,7 @@ async def get_dashboards( logger.error(f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}") raise HTTPException(status_code=404, detail="Environment not found") - bound_username: Optional[str] = None + bound_username: str | None = None can_apply_profile_filter = False can_apply_slug_filter = False effective_profile_filter = EffectiveProfileFilter( @@ -85,7 +92,7 @@ async def get_dashboards( username=None, match_logic=None, ) - profile_service: Optional[ProfileService] = None + profile_service: ProfileService | None = None try: profile_service_module = getattr(ProfileService, "__module__", "") @@ -245,7 +252,7 @@ async def get_dashboards( "Applying profile actor filter", extra={"src": "get_dashboards", "payload": {"env": env_id, "bound_username": bound_username, "actor_aliases": actor_aliases, "dashboards_before": len(dashboards)}}, ) - filtered_dashboards: List[Dict[str, Any]] = [] + filtered_dashboards: list[dict[str, Any]] = [] max_actor_samples = 15 for index, dashboard in enumerate(dashboards): owners_value = dashboard.get("owners") @@ -287,7 +294,7 @@ async def get_dashboards( or search_lower in d.get("slug", "").lower() ] - def _matches_dashboard_filters(dashboard: Dict[str, Any]) -> bool: + def _matches_dashboard_filters(dashboard: dict[str, Any]) -> bool: title_value = str(dashboard.get("title") or "").strip().lower() if title_filters and title_value not in title_filters: return False @@ -376,7 +383,7 @@ async def get_dashboards( f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}" ) raise HTTPException( - status_code=503, detail=f"Failed to fetch dashboards: {str(e)}" + status_code=503, detail=f"Failed to fetch dashboards: {e!s}" ) diff --git a/backend/src/api/routes/dashboards/_projection.py b/backend/src/api/routes/dashboards/_projection.py index 3b9c75b8..8b9730ba 100644 --- a/backend/src/api/routes/dashboards/_projection.py +++ b/backend/src/api/routes/dashboards/_projection.py @@ -4,9 +4,9 @@ # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [ProfileService] -from typing import Any, Dict, List, Optional +from typing import Any -from src.core.logger import logger, belief_scope +from src.core.logger import logger from src.core.superset_client import SupersetClient from src.core.superset_profile_lookup import SupersetAccountLookupAdapter from src.models.auth import User @@ -15,7 +15,7 @@ from src.services.profile_service import ProfileService # #region _normalize_actor_alias_token [C:2] [TYPE Function] # @BRIEF Normalize actor alias token to comparable trim+lower text. -def _normalize_actor_alias_token(value: Any) -> Optional[str]: +def _normalize_actor_alias_token(value: Any) -> str | None: if value is None: return None normalized = str(value).strip().lower() @@ -27,7 +27,7 @@ def _normalize_actor_alias_token(value: Any) -> Optional[str]: # #region _normalize_owner_display_token [C:2] [TYPE Function] # @BRIEF Project owner payload value into stable display string for API response contracts. -def _normalize_owner_display_token(owner: Any) -> Optional[str]: +def _normalize_owner_display_token(owner: Any) -> str | None: if owner is None: return None if isinstance(owner, dict): @@ -46,15 +46,15 @@ def _normalize_owner_display_token(owner: Any) -> Optional[str]: # #region _normalize_dashboard_owner_values [C:2] [TYPE Function] # @BRIEF Normalize dashboard owners payload to optional list of display strings. -def _normalize_dashboard_owner_values(owners: Any) -> Optional[List[str]]: +def _normalize_dashboard_owner_values(owners: Any) -> list[str] | None: if owners is None: return None - raw_items: List[Any] + raw_items: list[Any] if isinstance(owners, list): raw_items = owners else: raw_items = [owners] - normalized: List[str] = [] + normalized: list[str] = [] for owner in raw_items: token = _normalize_owner_display_token(owner) if token and token not in normalized: @@ -68,9 +68,9 @@ def _normalize_dashboard_owner_values(owners: Any) -> Optional[List[str]]: # #region _project_dashboard_response_items [C:2] [TYPE Function] # @BRIEF Project dashboard payloads to response-contract-safe shape. def _project_dashboard_response_items( - dashboards: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - projected: List[Dict[str, Any]] = [] + dashboards: list[dict[str, Any]], +) -> list[dict[str, Any]]: + projected: list[dict[str, Any]] = [] for dashboard in dashboards: projected_dashboard = dict(dashboard) projected_dashboard["owners"] = _normalize_dashboard_owner_values( @@ -87,9 +87,9 @@ def _project_dashboard_response_items( # @BRIEF Resolve dashboard profile-filter binding through current or legacy profile service contracts. def _get_profile_filter_binding( profile_service: Any, current_user: User -) -> Dict[str, Any]: +) -> dict[str, Any]: - def _read_optional_string(value: Any) -> Optional[str]: + def _read_optional_string(value: Any) -> str | None: return value if isinstance(value, str) else None def _read_bool(value: Any, default: bool) -> bool: @@ -143,11 +143,11 @@ def _get_profile_filter_binding( # #region _resolve_profile_actor_aliases [C:2] [TYPE Function] # @BRIEF Resolve stable actor aliases for profile filtering without per-dashboard detail fan-out. # @SIDE_EFFECT: Performs at most one Superset users-lookup request. -def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: +def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> list[str]: normalized_bound = _normalize_actor_alias_token(bound_username) if not normalized_bound: return [] - aliases: List[str] = [normalized_bound] + aliases: list[str] = [normalized_bound] try: client = SupersetClient(env) adapter = SupersetAccountLookupAdapter( @@ -164,7 +164,7 @@ def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: lookup_items = ( lookup_payload.get("items", []) if isinstance(lookup_payload, dict) else [] ) - matched_item: Optional[Dict[str, Any]] = None + matched_item: dict[str, Any] | None = None for item in lookup_items: if not isinstance(item, dict): continue @@ -200,9 +200,9 @@ def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: # @BRIEF Apply profile actor matching against multiple aliases (username + optional display name). def _matches_dashboard_actor_aliases( profile_service: ProfileService, - actor_aliases: List[str], - owners: Optional[Any], - modified_by: Optional[str], + actor_aliases: list[str], + owners: Any | None, + modified_by: str | None, ) -> bool: for actor_alias in actor_aliases: if profile_service.matches_dashboard_actor( @@ -220,7 +220,7 @@ def _matches_dashboard_actor_aliases( # #region _task_matches_dashboard [C:2] [TYPE Function] # @BRIEF Checks whether task params are tied to a specific dashboard and environment. def _task_matches_dashboard( - task: Any, dashboard_id: int, env_id: Optional[str] + task: Any, dashboard_id: int, env_id: str | None ) -> bool: plugin_id = getattr(task, "plugin_id", None) if plugin_id not in {"superset-backup", "llm_dashboard_validation"}: diff --git a/backend/src/api/routes/dashboards/_schemas.py b/backend/src/api/routes/dashboards/_schemas.py index 568ac9fa..728666d6 100644 --- a/backend/src/api/routes/dashboards/_schemas.py +++ b/backend/src/api/routes/dashboards/_schemas.py @@ -3,17 +3,18 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> [Pydantic] -from typing import List, Optional, Dict, Any, Literal +from typing import Literal + from pydantic import BaseModel, Field # #region GitStatus [C:1] [TYPE DataClass] # @BRIEF DTO for dashboard Git synchronization status. class GitStatus(BaseModel): - branch: Optional[str] = None - sync_status: Optional[str] = Field(None, pattern="^OK|DIFF|NO_REPO|ERROR$") - has_repo: Optional[bool] = None - has_changes_for_commit: Optional[bool] = None + branch: str | None = None + sync_status: str | None = Field(None, pattern="^OK|DIFF|NO_REPO|ERROR$") + has_repo: bool | None = None + has_changes_for_commit: bool | None = None # #endregion GitStatus @@ -22,12 +23,12 @@ class GitStatus(BaseModel): # #region LastTask [C:2] [TYPE DataClass] # @BRIEF DTO for the most recent background task associated with a dashboard. class LastTask(BaseModel): - task_id: Optional[str] = None - status: Optional[str] = Field( + task_id: str | None = None + status: str | None = Field( None, pattern="^PENDING|RUNNING|SUCCESS|FAILED|ERROR|AWAITING_INPUT|WAITING_INPUT|AWAITING_MAPPING$", ) - validation_status: Optional[str] = Field(None, pattern="^PASS|FAIL|WARN|UNKNOWN$") + validation_status: str | None = Field(None, pattern="^PASS|FAIL|WARN|UNKNOWN$") # #endregion LastTask @@ -38,14 +39,14 @@ class LastTask(BaseModel): class DashboardItem(BaseModel): id: int title: str - slug: Optional[str] = None - url: Optional[str] = None - last_modified: Optional[str] = None - created_by: Optional[str] = None - modified_by: Optional[str] = None - owners: Optional[List[str]] = None - git_status: Optional[GitStatus] = None - last_task: Optional[LastTask] = None + slug: str | None = None + url: str | None = None + last_modified: str | None = None + created_by: str | None = None + modified_by: str | None = None + owners: list[str] | None = None + git_status: GitStatus | None = None + last_task: LastTask | None = None # #endregion DashboardItem @@ -57,10 +58,8 @@ class EffectiveProfileFilter(BaseModel): applied: bool source_page: Literal["dashboards_main", "other"] = "dashboards_main" override_show_all: bool = False - username: Optional[str] = None - match_logic: Optional[ - Literal["owners_or_modified_by", "slug_only", "owners_or_modified_by+slug_only"] - ] = None + username: str | None = None + match_logic: Literal["owners_or_modified_by", "slug_only", "owners_or_modified_by+slug_only"] | None = None # #endregion EffectiveProfileFilter @@ -69,12 +68,12 @@ class EffectiveProfileFilter(BaseModel): # #region DashboardsResponse [C:1] [TYPE DataClass] # @BRIEF Envelope DTO for paginated dashboards list. class DashboardsResponse(BaseModel): - dashboards: List[DashboardItem] + dashboards: list[DashboardItem] total: int page: int page_size: int total_pages: int - effective_profile_filter: Optional[EffectiveProfileFilter] = None + effective_profile_filter: EffectiveProfileFilter | None = None # #endregion DashboardsResponse @@ -85,10 +84,10 @@ class DashboardsResponse(BaseModel): class DashboardChartItem(BaseModel): id: int title: str - viz_type: Optional[str] = None - dataset_id: Optional[int] = None - last_modified: Optional[str] = None - overview: Optional[str] = None + viz_type: str | None = None + dataset_id: int | None = None + last_modified: str | None = None + overview: str | None = None # #endregion DashboardChartItem @@ -99,10 +98,10 @@ class DashboardChartItem(BaseModel): class DashboardDatasetItem(BaseModel): id: int table_name: str - schema: Optional[str] = None + schema: str | None = None database: str - last_modified: Optional[str] = None - overview: Optional[str] = None + last_modified: str | None = None + overview: str | None = None # #endregion DashboardDatasetItem @@ -113,13 +112,13 @@ class DashboardDatasetItem(BaseModel): class DashboardDetailResponse(BaseModel): id: int title: str - slug: Optional[str] = None - url: Optional[str] = None - description: Optional[str] = None - last_modified: Optional[str] = None - published: Optional[bool] = None - charts: List[DashboardChartItem] - datasets: List[DashboardDatasetItem] + slug: str | None = None + url: str | None = None + description: str | None = None + last_modified: str | None = None + published: bool | None = None + charts: list[DashboardChartItem] + datasets: list[DashboardDatasetItem] chart_count: int dataset_count: int @@ -133,11 +132,11 @@ class DashboardTaskHistoryItem(BaseModel): id: str plugin_id: str status: str - validation_status: Optional[str] = None - started_at: Optional[str] = None - finished_at: Optional[str] = None - env_id: Optional[str] = None - summary: Optional[str] = None + validation_status: str | None = None + started_at: str | None = None + finished_at: str | None = None + env_id: str | None = None + summary: str | None = None # #endregion DashboardTaskHistoryItem @@ -147,7 +146,7 @@ class DashboardTaskHistoryItem(BaseModel): # @BRIEF Collection DTO for task history. class DashboardTaskHistoryResponse(BaseModel): dashboard_id: int - items: List[DashboardTaskHistoryItem] + items: list[DashboardTaskHistoryItem] # #endregion DashboardTaskHistoryResponse @@ -158,8 +157,8 @@ class DashboardTaskHistoryResponse(BaseModel): class DatabaseMapping(BaseModel): source_db: str target_db: str - source_db_uuid: Optional[str] = None - target_db_uuid: Optional[str] = None + source_db_uuid: str | None = None + target_db_uuid: str | None = None confidence: float @@ -169,7 +168,7 @@ class DatabaseMapping(BaseModel): # #region DatabaseMappingsResponse [C:1] [TYPE DataClass] # @BRIEF Wrapper for database mappings. class DatabaseMappingsResponse(BaseModel): - mappings: List[DatabaseMapping] + mappings: list[DatabaseMapping] # #endregion DatabaseMappingsResponse @@ -180,10 +179,10 @@ class DatabaseMappingsResponse(BaseModel): class MigrateRequest(BaseModel): source_env_id: str = Field(..., description="Source environment ID") target_env_id: str = Field(..., description="Target environment ID") - dashboard_ids: List[int] = Field( + dashboard_ids: list[int] = Field( ..., description="List of dashboard IDs to migrate" ) - db_mappings: Optional[Dict[str, str]] = Field( + db_mappings: dict[str, str] | None = Field( None, description="Database mappings for migration" ) replace_db_config: bool = Field(False, description="Replace database configuration") @@ -205,8 +204,8 @@ class TaskResponse(BaseModel): # @BRIEF DTO for dashboard backup requests. class BackupRequest(BaseModel): env_id: str = Field(..., description="Environment ID") - dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to backup") - schedule: Optional[str] = Field( + dashboard_ids: list[int] = Field(..., description="List of dashboard IDs to backup") + schedule: str | None = Field( None, description="Cron schedule for recurring backups (e.g., '0 0 * * *')" ) diff --git a/backend/src/api/routes/dataset_review.py b/backend/src/api/routes/dataset_review.py index 6d71bc7e..58bd9762 100644 --- a/backend/src/api/routes/dataset_review.py +++ b/backend/src/api/routes/dataset_review.py @@ -5,31 +5,31 @@ # @REJECTED: Keeping all routes in one file because it exceeded the fractal limit by 6x and accumulated severe structural erosion risk. from src.api.routes.dataset_review_pkg._dependencies import ( # noqa: F401 - StartSessionRequest, - UpdateSessionRequest, - SessionCollectionResponse, - ExportArtifactResponse, - FieldSemanticUpdateRequest, - FeedbackRequest, - ClarificationAnswerRequest, - ClarificationSessionSummaryResponse, - ClarificationStateResponse, - ClarificationAnswerResultResponse, - FeedbackResponse, ApproveMappingRequest, + BatchApproveMappingRequest, BatchApproveSemanticItemRequest, BatchApproveSemanticRequest, - BatchApproveMappingRequest, - PreviewEnqueueResultResponse, - MappingCollectionResponse, - UpdateExecutionMappingRequest, + ClarificationAnswerRequest, + ClarificationAnswerResultResponse, + ClarificationSessionSummaryResponse, + ClarificationStateResponse, + ExportArtifactResponse, + FeedbackRequest, + FeedbackResponse, + FieldSemanticUpdateRequest, LaunchDatasetResponse, + MappingCollectionResponse, + PreviewEnqueueResultResponse, + SessionCollectionResponse, + StartSessionRequest, + UpdateExecutionMappingRequest, + UpdateSessionRequest, + _get_clarification_engine, + _get_orchestrator, + _get_repository, _require_auto_review_flag, _require_clarification_flag, _require_execution_flag, - _get_repository, - _get_orchestrator, - _get_clarification_engine, _update_semantic_field_state, ) from src.api.routes.dataset_review_pkg._routes import router # noqa: F401 diff --git a/backend/src/api/routes/dataset_review_pkg/_dependencies.py b/backend/src/api/routes/dataset_review_pkg/_dependencies.py index f7f495c1..0b5806e2 100644 --- a/backend/src/api/routes/dataset_review_pkg/_dependencies.py +++ b/backend/src/api/routes/dataset_review_pkg/_dependencies.py @@ -4,11 +4,10 @@ from __future__ import annotations -import json from datetime import datetime -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any -from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status +from fastapi import Depends, Header, HTTPException, status from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -16,33 +15,18 @@ from src.core.database import get_db from src.core.logger import belief_scope, logger from src.dependencies import ( get_config_manager, - get_current_user, get_task_manager, - has_permission, ) -from src.models.auth import User from src.models.dataset_review import ( AnswerKind, - ApprovalState, ArtifactFormat, CandidateStatus, - ClarificationSession, DatasetReviewSession, - ExecutionMapping, FieldProvenance, - MappingMethod, - PreviewStatus, - QuestionState, - ReadinessState, - RecommendedAction, - SemanticCandidate, - SemanticFieldEntry, SessionStatus, ) from src.schemas.dataset_review import ( - ClarificationAnswerDto, ClarificationQuestionDto, - ClarificationSessionDto, CompiledPreviewDto, DatasetRunContextDto, ExecutionMappingDto, @@ -52,16 +36,10 @@ from src.schemas.dataset_review import ( ValidationFindingDto, ) from src.services.dataset_review.clarification_engine import ( - ClarificationAnswerCommand, ClarificationEngine, - ClarificationQuestionPayload, - ClarificationStateResult, ) from src.services.dataset_review.orchestrator import ( DatasetReviewOrchestrator, - LaunchDatasetCommand, - PreparePreviewCommand, - StartSessionCommand, ) from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionRepository, @@ -84,7 +62,7 @@ class StartSessionRequest(BaseModel): # @BRIEF Request DTO for lifecycle state updates on an existing session. class UpdateSessionRequest(BaseModel): status: SessionStatus - note: Optional[str] = None + note: str | None = None # #endregion UpdateSessionRequest @@ -93,7 +71,7 @@ class UpdateSessionRequest(BaseModel): # #region SessionCollectionResponse [C:1] [TYPE Class] # @BRIEF Paginated session collection response. class SessionCollectionResponse(BaseModel): - items: List[SessionSummary] + items: list[SessionSummary] total: int page: int page_size: int @@ -112,8 +90,8 @@ class ExportArtifactResponse(BaseModel): format: str storage_ref: str created_by_user_id: str - created_at: Optional[str] = None - content: Dict[str, Any] + created_at: str | None = None + content: dict[str, Any] # #endregion ExportArtifactResponse @@ -122,12 +100,12 @@ class ExportArtifactResponse(BaseModel): # #region FieldSemanticUpdateRequest [C:1] [TYPE Class] # @BRIEF Request DTO for field-level semantic candidate acceptance or manual override. class FieldSemanticUpdateRequest(BaseModel): - candidate_id: Optional[str] = None - verbose_name: Optional[str] = None - description: Optional[str] = None - display_format: Optional[str] = None + candidate_id: str | None = None + verbose_name: str | None = None + description: str | None = None + display_format: str | None = None lock_field: bool = False - resolution_note: Optional[str] = None + resolution_note: str | None = None # #endregion FieldSemanticUpdateRequest @@ -147,7 +125,7 @@ class FeedbackRequest(BaseModel): class ClarificationAnswerRequest(BaseModel): question_id: str = Field(..., min_length=1) answer_kind: AnswerKind - answer_value: Optional[str] = None + answer_value: str | None = None # #endregion ClarificationAnswerRequest @@ -159,10 +137,10 @@ class ClarificationSessionSummaryResponse(BaseModel): clarification_session_id: str session_id: str status: str - current_question_id: Optional[str] = None + current_question_id: str | None = None resolved_count: int remaining_count: int - summary_delta: Optional[str] = None + summary_delta: str | None = None # #endregion ClarificationSessionSummaryResponse @@ -171,8 +149,8 @@ class ClarificationSessionSummaryResponse(BaseModel): # #region ClarificationStateResponse [C:1] [TYPE Class] # @BRIEF Response DTO for current clarification state and active question payload. class ClarificationStateResponse(BaseModel): - clarification_session: Optional[ClarificationSessionSummaryResponse] = None - current_question: Optional[ClarificationQuestionDto] = None + clarification_session: ClarificationSessionSummaryResponse | None = None + current_question: ClarificationQuestionDto | None = None # #endregion ClarificationStateResponse @@ -183,7 +161,7 @@ class ClarificationStateResponse(BaseModel): class ClarificationAnswerResultResponse(BaseModel): clarification_state: ClarificationStateResponse session: SessionSummary - changed_findings: List[ValidationFindingDto] + changed_findings: list[ValidationFindingDto] # #endregion ClarificationAnswerResultResponse @@ -202,7 +180,7 @@ class FeedbackResponse(BaseModel): # #region ApproveMappingRequest [C:1] [TYPE Class] # @BRIEF Optional request DTO for explicit mapping approval audit notes. class ApproveMappingRequest(BaseModel): - approval_note: Optional[str] = None + approval_note: str | None = None # #endregion ApproveMappingRequest @@ -222,7 +200,7 @@ class BatchApproveSemanticItemRequest(BaseModel): # #region BatchApproveSemanticRequest [C:1] [TYPE Class] # @BRIEF Request DTO for explicit batch semantic approvals. class BatchApproveSemanticRequest(BaseModel): - items: List[BatchApproveSemanticItemRequest] = Field(..., min_length=1) + items: list[BatchApproveSemanticItemRequest] = Field(..., min_length=1) # #endregion BatchApproveSemanticRequest @@ -231,8 +209,8 @@ class BatchApproveSemanticRequest(BaseModel): # #region BatchApproveMappingRequest [C:1] [TYPE Class] # @BRIEF Request DTO for explicit batch mapping approvals. class BatchApproveMappingRequest(BaseModel): - mapping_ids: List[str] = Field(..., min_length=1) - approval_note: Optional[str] = None + mapping_ids: list[str] = Field(..., min_length=1) + approval_note: str | None = None # #endregion BatchApproveMappingRequest @@ -242,9 +220,9 @@ class BatchApproveMappingRequest(BaseModel): # @BRIEF Async preview trigger response exposing only enqueue state. class PreviewEnqueueResultResponse(BaseModel): session_id: str - session_version: Optional[int] = None + session_version: int | None = None preview_status: str - task_id: Optional[str] = None + task_id: str | None = None # #endregion PreviewEnqueueResultResponse @@ -253,7 +231,7 @@ class PreviewEnqueueResultResponse(BaseModel): # #region MappingCollectionResponse [C:1] [TYPE Class] # @BRIEF Wrapper for execution mapping list responses. class MappingCollectionResponse(BaseModel): - items: List[ExecutionMappingDto] + items: list[ExecutionMappingDto] # #endregion MappingCollectionResponse @@ -262,9 +240,9 @@ class MappingCollectionResponse(BaseModel): # #region UpdateExecutionMappingRequest [C:1] [TYPE Class] # @BRIEF Request DTO for one manual execution-mapping override update. class UpdateExecutionMappingRequest(BaseModel): - effective_value: Optional[Any] = None - mapping_method: Optional[str] = Field(default=None, pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$") - transformation_note: Optional[str] = None + effective_value: Any | None = None + mapping_method: str | None = Field(default=None, pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$") + transformation_note: str | None = None # #endregion UpdateExecutionMappingRequest diff --git a/backend/src/api/routes/dataset_review_pkg/_routes.py b/backend/src/api/routes/dataset_review_pkg/_routes.py index f04fdce6..4395c406 100644 --- a/backend/src/api/routes/dataset_review_pkg/_routes.py +++ b/backend/src/api/routes/dataset_review_pkg/_routes.py @@ -5,43 +5,10 @@ from __future__ import annotations from datetime import datetime -from typing import Any, List, Optional, Union, cast +from typing import Any, Union, cast from fastapi import APIRouter, Depends, HTTPException, Query, Response, status -from src.core.logger import belief_scope, logger -from src.dependencies import get_config_manager, get_current_user, has_permission -from src.models.auth import User -from src.models.dataset_review import ( - ApprovalState, - ArtifactFormat, - FieldProvenance, - MappingMethod, - PreviewStatus, - ReadinessState, - RecommendedAction, - SessionStatus, -) -from src.schemas.dataset_review import ( - ClarificationAnswerDto, - CompiledPreviewDto, - ExecutionMappingDto, - SemanticFieldEntryDto, - SessionSummary, - ValidationFindingDto, -) -from src.services.dataset_review.clarification_engine import ( - ClarificationAnswerCommand, - ClarificationStateResult, -) -from src.services.dataset_review.orchestrator import ( - LaunchDatasetCommand, - PreparePreviewCommand, - StartSessionCommand, -) -from src.services.dataset_review.repositories.session_repository import ( - DatasetReviewSessionVersionConflictError, -) from src.api.routes.dataset_review_pkg._dependencies import ( BatchApproveMappingRequest, BatchApproveSemanticRequest, @@ -60,15 +27,16 @@ from src.api.routes.dataset_review_pkg._dependencies import ( UpdateExecutionMappingRequest, UpdateSessionRequest, _build_documentation_export, + _build_session_version_conflict_http_exception, _build_sql_lab_redirect_url, _build_validation_export, _commit_owned_session_mutation, _get_clarification_engine, _get_latest_clarification_session_or_404, + _get_orchestrator, _get_owned_field_or_404, _get_owned_mapping_or_404, _get_owned_session_or_404, - _get_orchestrator, _get_repository, _prepare_owned_session_mutation, _record_session_event, @@ -85,7 +53,38 @@ from src.api.routes.dataset_review_pkg._dependencies import ( _serialize_session_detail, _serialize_session_summary, _update_semantic_field_state, - _build_session_version_conflict_http_exception, +) +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.dataset_review import ( + ApprovalState, + ArtifactFormat, + FieldProvenance, + MappingMethod, + PreviewStatus, + ReadinessState, + RecommendedAction, + SessionStatus, +) +from src.schemas.dataset_review import ( + CompiledPreviewDto, + ExecutionMappingDto, + SemanticFieldEntryDto, + SessionSummary, + ValidationFindingDto, +) +from src.services.dataset_review.clarification_engine import ( + ClarificationAnswerCommand, + ClarificationStateResult, +) +from src.services.dataset_review.orchestrator import ( + LaunchDatasetCommand, + PreparePreviewCommand, + StartSessionCommand, +) +from src.services.dataset_review.repositories.session_repository import ( + DatasetReviewSessionVersionConflictError, ) router = APIRouter(prefix="/api/dataset-orchestration", tags=["Dataset Orchestration"]) @@ -639,7 +638,7 @@ async def unlock_field_semantic( # @BRIEF Approve multiple semantic candidate decisions in one batch. @router.post( "/sessions/{session_id}/fields/semantic/approve-batch", - response_model=List[SemanticFieldEntryDto], + response_model=list[SemanticFieldEntryDto], dependencies=[ Depends(_require_auto_review_flag), Depends(has_permission("dataset:session", "MANAGE")), @@ -828,7 +827,7 @@ async def approve_execution_mapping( # @BRIEF Approve multiple warning-sensitive execution mappings in one batch. @router.post( "/sessions/{session_id}/mappings/approve-batch", - response_model=List[ExecutionMappingDto], + response_model=list[ExecutionMappingDto], dependencies=[ Depends(_require_auto_review_flag), Depends(_require_execution_flag), diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 1108dcdd..08e1b14f 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -8,12 +8,13 @@ # # @INVARIANT: All dataset responses include last_task metadata -from fastapi import APIRouter, Depends, HTTPException, Query -from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field -from ...dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission -from ...core.logger import logger, belief_scope + +from ...core.logger import belief_scope, logger from ...core.superset_client import SupersetClient +from ...dependencies import get_config_manager, get_resource_service, get_task_manager, has_permission router = APIRouter(prefix="/api/datasets", tags=["Datasets"]) @@ -27,8 +28,8 @@ class MappedFields(BaseModel): # #region LastTask [C:1] [TYPE DataClass] # @BRIEF DTO for the most recent task associated with a dataset class LastTask(BaseModel): - task_id: Optional[str] = None - status: Optional[str] = Field(None, pattern="^RUNNING|SUCCESS|ERROR|WAITING_INPUT$") + task_id: str | None = None + status: str | None = Field(None, pattern="^RUNNING|SUCCESS|ERROR|WAITING_INPUT$") # #endregion LastTask # #region DatasetItem [C:1] [TYPE DataClass] @@ -38,8 +39,8 @@ class DatasetItem(BaseModel): table_name: str schema_name: str = Field(..., alias="schema") database: str - mapped_fields: Optional[MappedFields] = None - last_task: Optional[LastTask] = None + mapped_fields: MappedFields | None = None + last_task: LastTask | None = None class Config: allow_population_by_field_name = True @@ -50,7 +51,7 @@ class DatasetItem(BaseModel): class LinkedDashboard(BaseModel): id: int title: str - slug: Optional[str] = None + slug: str | None = None # #endregion LinkedDashboard # #region DatasetColumn [C:1] [TYPE DataClass] @@ -58,28 +59,28 @@ class LinkedDashboard(BaseModel): class DatasetColumn(BaseModel): id: int name: str - type: Optional[str] = None + type: str | None = None is_dttm: bool = False is_active: bool = True - description: Optional[str] = None + description: str | None = None # #endregion DatasetColumn # #region DatasetDetailResponse [C:1] [TYPE DataClass] # @BRIEF Detailed DTO for a dataset including columns and links class DatasetDetailResponse(BaseModel): id: int - table_name: Optional[str] = None - schema_name: Optional[str] = Field(None, alias="schema") + table_name: str | None = None + schema_name: str | None = Field(None, alias="schema") database: str - description: Optional[str] = None - columns: List[DatasetColumn] + description: str | None = None + columns: list[DatasetColumn] column_count: int - sql: Optional[str] = None - linked_dashboards: List[LinkedDashboard] + sql: str | None = None + linked_dashboards: list[LinkedDashboard] linked_dashboard_count: int is_sqllab_view: bool = False - created_on: Optional[str] = None - changed_on: Optional[str] = None + created_on: str | None = None + changed_on: str | None = None class Config: allow_population_by_field_name = True @@ -88,7 +89,7 @@ class DatasetDetailResponse(BaseModel): # #region DatasetsResponse [C:1] [TYPE DataClass] # @BRIEF Paginated response DTO for dataset listings class DatasetsResponse(BaseModel): - datasets: List[DatasetItem] + datasets: list[DatasetItem] total: int page: int page_size: int @@ -109,7 +110,7 @@ class TaskResponse(BaseModel): @router.get("/ids") async def get_dataset_ids( env_id: str, - search: Optional[str] = None, + search: str | None = None, config_manager=Depends(get_config_manager), task_manager=Depends(get_task_manager), resource_service=Depends(get_resource_service), @@ -122,31 +123,31 @@ async def get_dataset_ids( if not env: logger.error(f"[get_dataset_ids][Coherence:Failed] Environment not found: {env_id}") raise HTTPException(status_code=404, detail="Environment not found") - + try: # Get all tasks for status lookup all_tasks = task_manager.get_all_tasks() - + # Fetch datasets with status using ResourceService datasets = await resource_service.get_datasets_with_status(env, all_tasks) - + # Apply search filter if provided if search: search_lower = search.lower() datasets = [ - d for d in datasets + d for d in datasets if search_lower in d.get('table_name', '').lower() ] - + # Extract and return just the IDs dataset_ids = [d['id'] for d in datasets] logger.info(f"[get_dataset_ids][Coherence:OK] Returning {len(dataset_ids)} dataset IDs") - + return {"dataset_ids": dataset_ids} - + except Exception as e: logger.error(f"[get_dataset_ids][Coherence:Failed] Failed to fetch dataset IDs: {e}") - raise HTTPException(status_code=503, detail=f"Failed to fetch dataset IDs: {str(e)}") + raise HTTPException(status_code=503, detail=f"Failed to fetch dataset IDs: {e!s}") # #endregion get_dataset_ids # #region get_datasets [C:3] [TYPE Function] @@ -160,7 +161,7 @@ async def get_dataset_ids( @router.get("", response_model=DatasetsResponse) async def get_datasets( env_id: str, - search: Optional[str] = None, + search: str | None = None, page: int = 1, page_size: int = 10, config_manager=Depends(get_config_manager), @@ -176,40 +177,40 @@ async def get_datasets( if page_size < 1 or page_size > 100: logger.error(f"[get_datasets][Coherence:Failed] Invalid page_size: {page_size}") raise HTTPException(status_code=400, detail="Page size must be between 1 and 100") - + # Validate environment exists environments = config_manager.get_environments() env = next((e for e in environments if e.id == env_id), None) if not env: logger.error(f"[get_datasets][Coherence:Failed] Environment not found: {env_id}") raise HTTPException(status_code=404, detail="Environment not found") - + try: # Get all tasks for status lookup all_tasks = task_manager.get_all_tasks() - + # Fetch datasets with status using ResourceService datasets = await resource_service.get_datasets_with_status(env, all_tasks) - + # Apply search filter if provided if search: search_lower = search.lower() datasets = [ - d for d in datasets + d for d in datasets if search_lower in d.get('table_name', '').lower() ] - + # Calculate pagination total = len(datasets) total_pages = (total + page_size - 1) // page_size if total > 0 else 1 start_idx = (page - 1) * page_size end_idx = start_idx + page_size - + # Slice datasets for current page paginated_datasets = datasets[start_idx:end_idx] - + logger.info(f"[get_datasets][Coherence:OK] Returning {len(paginated_datasets)} datasets (page {page}/{total_pages}, total: {total})") - + return DatasetsResponse( datasets=paginated_datasets, total=total, @@ -217,20 +218,20 @@ async def get_datasets( page_size=page_size, total_pages=total_pages ) - + except Exception as e: logger.error(f"[get_datasets][Coherence:Failed] Failed to fetch datasets: {e}") - raise HTTPException(status_code=503, detail=f"Failed to fetch datasets: {str(e)}") + raise HTTPException(status_code=503, detail=f"Failed to fetch datasets: {e!s}") # #endregion get_datasets # #region MapColumnsRequest [C:1] [TYPE DataClass] # @BRIEF Request DTO for initiating column mapping class MapColumnsRequest(BaseModel): env_id: str = Field(..., description="Environment ID") - dataset_ids: List[int] = Field(..., description="List of dataset IDs to map") + dataset_ids: list[int] = Field(..., description="List of dataset IDs to map") source_type: str = Field(..., description="Source type: 'postgresql' or 'xlsx'") - connection_id: Optional[str] = Field(None, description="Connection ID for PostgreSQL source") - file_data: Optional[str] = Field(None, description="File path or data for XLSX source") + connection_id: str | None = Field(None, description="Connection ID for PostgreSQL source") + file_data: str | None = Field(None, description="File path or data for XLSX source") # #endregion MapColumnsRequest # #region map_columns [C:3] [TYPE Function] @@ -254,20 +255,20 @@ async def map_columns( if not request.dataset_ids: logger.error("[map_columns][Coherence:Failed] No dataset IDs provided") raise HTTPException(status_code=400, detail="At least one dataset ID must be provided") - + # Validate source type if request.source_type not in ['postgresql', 'xlsx']: logger.error(f"[map_columns][Coherence:Failed] Invalid source type: {request.source_type}") raise HTTPException(status_code=400, detail="Source type must be 'postgresql' or 'xlsx'") - + # Validate environment exists environments = config_manager.get_environments() env = next((e for e in environments if e.id == request.env_id), None) - + if not env: logger.error(f"[map_columns][Coherence:Failed] Environment not found: {request.env_id}") raise HTTPException(status_code=404, detail="Environment not found") - + try: # Create mapping task task_params = { @@ -277,28 +278,28 @@ async def map_columns( 'connection_id': request.connection_id, 'file_data': request.file_data } - + task_obj = await task_manager.create_task( plugin_id='dataset-mapper', params=task_params ) - + logger.info(f"[map_columns][Coherence:OK] Mapping task created: {task_obj.id} for {len(request.dataset_ids)} datasets") - + return TaskResponse(task_id=str(task_obj.id)) - + except Exception as e: logger.error(f"[map_columns][Coherence:Failed] Failed to create mapping task: {e}") - raise HTTPException(status_code=503, detail=f"Failed to create mapping task: {str(e)}") + raise HTTPException(status_code=503, detail=f"Failed to create mapping task: {e!s}") # #endregion map_columns # #region GenerateDocsRequest [C:1] [TYPE DataClass] # @BRIEF Request DTO for initiating documentation generation class GenerateDocsRequest(BaseModel): env_id: str = Field(..., description="Environment ID") - dataset_ids: List[int] = Field(..., description="List of dataset IDs to generate docs for") + dataset_ids: list[int] = Field(..., description="List of dataset IDs to generate docs for") llm_provider: str = Field(..., description="LLM provider to use") - options: Optional[dict] = Field(None, description="Additional options for documentation generation") + options: dict | None = Field(None, description="Additional options for documentation generation") # #endregion GenerateDocsRequest # #region generate_docs [C:3] [TYPE Function] @@ -322,15 +323,15 @@ async def generate_docs( if not request.dataset_ids: logger.error("[generate_docs][Coherence:Failed] No dataset IDs provided") raise HTTPException(status_code=400, detail="At least one dataset ID must be provided") - + # Validate environment exists environments = config_manager.get_environments() env = next((e for e in environments if e.id == request.env_id), None) - + if not env: logger.error(f"[generate_docs][Coherence:Failed] Environment not found: {request.env_id}") raise HTTPException(status_code=404, detail="Environment not found") - + try: # Create documentation generation task task_params = { @@ -339,19 +340,19 @@ async def generate_docs( 'provider_id': request.llm_provider, 'options': request.options or {} } - + task_obj = await task_manager.create_task( plugin_id='llm_documentation', params=task_params ) - + logger.info(f"[generate_docs][Coherence:OK] Documentation generation task created: {task_obj.id} for {len(request.dataset_ids)} datasets") - + return TaskResponse(task_id=str(task_obj.id)) - + except Exception as e: logger.error(f"[generate_docs][Coherence:Failed] Failed to create documentation generation task: {e}") - raise HTTPException(status_code=503, detail=f"Failed to create documentation generation task: {str(e)}") + raise HTTPException(status_code=503, detail=f"Failed to create documentation generation task: {e!s}") # #endregion generate_docs # #region get_dataset_detail [C:3] [TYPE Function] @@ -374,19 +375,19 @@ async def get_dataset_detail( if not env: logger.error(f"[get_dataset_detail][Coherence:Failed] Environment not found: {env_id}") raise HTTPException(status_code=404, detail="Environment not found") - + try: # Fetch detailed dataset info using SupersetClient client = SupersetClient(env) dataset_detail = client.get_dataset_detail(dataset_id) - + logger.info(f"[get_dataset_detail][Coherence:OK] Retrieved dataset {dataset_id} with {dataset_detail['column_count']} columns and {dataset_detail['linked_dashboard_count']} linked dashboards") - + return DatasetDetailResponse(**dataset_detail) - + except Exception as e: logger.error(f"[get_dataset_detail][Coherence:Failed] Failed to fetch dataset detail: {e}") - raise HTTPException(status_code=503, detail=f"Failed to fetch dataset detail: {str(e)}") + raise HTTPException(status_code=503, detail=f"Failed to fetch dataset detail: {e!s}") # #endregion get_dataset_detail # #endregion DatasetsApi diff --git a/backend/src/api/routes/environments.py b/backend/src/api/routes/environments.py index c25d9eb4..2186fb72 100644 --- a/backend/src/api/routes/environments.py +++ b/backend/src/api/routes/environments.py @@ -7,12 +7,13 @@ # # @INVARIANT: Environment IDs must exist in the configuration. + from fastapi import APIRouter, Depends, HTTPException -from typing import List, Optional -from ...dependencies import get_config_manager, get_scheduler_service, has_permission -from ...core.superset_client import SupersetClient from pydantic import BaseModel, Field + from ...core.logger import belief_scope +from ...core.superset_client import SupersetClient +from ...dependencies import get_config_manager, get_scheduler_service, has_permission router = APIRouter(prefix="/api/environments", tags=["Environments"]) @@ -41,14 +42,14 @@ class EnvironmentResponse(BaseModel): url: str stage: str = "DEV" is_production: bool = False - backup_schedule: Optional[ScheduleSchema] = None + backup_schedule: ScheduleSchema | None = None # #endregion EnvironmentResponse # #region DatabaseResponse [TYPE DataClass] class DatabaseResponse(BaseModel): uuid: str database_name: str - engine: Optional[str] + engine: str | None # #endregion DatabaseResponse # #region get_environments [TYPE Function] [SEMANTICS list, environments, config] @@ -56,7 +57,7 @@ class DatabaseResponse(BaseModel): # @LAYER: API # @PRE: config_manager is injected via Depends. # @POST: Returns a list of EnvironmentResponse objects. -@router.get("", response_model=List[EnvironmentResponse]) +@router.get("", response_model=list[EnvironmentResponse]) async def get_environments( config_manager=Depends(get_config_manager), _ = Depends(has_permission("environments", "READ")) @@ -106,16 +107,16 @@ async def update_environment_schedule( env = next((e for e in envs if e.id == id), None) if not env: raise HTTPException(status_code=404, detail="Environment not found") - + # Update environment config env.backup_schedule.enabled = schedule.enabled env.backup_schedule.cron_expression = schedule.cron_expression - + config_manager.update_environment(id, env) - + # Refresh scheduler scheduler_service.load_schedules() - + return {"message": "Schedule updated successfully"} # #endregion update_environment_schedule @@ -135,13 +136,13 @@ async def get_environment_databases( env = next((e for e in envs if e.id == id), None) if not env: raise HTTPException(status_code=404, detail="Environment not found") - + try: # Initialize SupersetClient from environment config client = SupersetClient(env) return client.get_databases_summary() except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to fetch databases: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to fetch databases: {e!s}") # #endregion get_environment_databases # #endregion EnvironmentsApi diff --git a/backend/src/api/routes/git/__init__.py b/backend/src/api/routes/git/__init__.py index 51192352..34ca4726 100644 --- a/backend/src/api/routes/git/__init__.py +++ b/backend/src/api/routes/git/__init__.py @@ -11,32 +11,32 @@ import os from src.services.git_service import GitService -from ._router import router from ._deps import MAX_REPOSITORY_STATUS_BATCH +from ._router import router # -- git_service singleton (canonical instance; tests may monkeypatch git_routes.git_service) -- git_service = GitService() # -- Config routes -- -from ._config_routes import get_git_configs, create_git_config, update_git_config, delete_git_config, test_git_config # noqa: E501, F401 - -# -- Gitea routes -- -from ._gitea_routes import list_gitea_repositories, create_gitea_repository, delete_gitea_repository, create_remote_repository # noqa: E501, F401 - -# -- Repo routes (core) -- -from ._repo_routes import init_repository, get_repository_binding, delete_repository, get_branches, create_branch, checkout_branch # noqa: E501, F401 -from ._helpers import _resolve_dashboard_id_from_ref, _resolve_dashboard_id_from_ref_async, _resolve_repo_key_from_ref # noqa: E501, F401 — re-exported for test monkeypatch compatibility - -# -- Repo operations routes (commit, push, pull, status, diff, history, generate-message) -- -from ._repo_operations_routes import commit_changes, push_changes, pull_changes, get_repository_status, get_repository_status_batch, get_repository_diff, get_history, generate_commit_message # noqa: E501, F401 - -# -- Repo lifecycle routes (sync, promote, deploy) -- -from ._repo_lifecycle_routes import sync_dashboard, promote_dashboard, deploy_dashboard # noqa: E501, F401 - -# -- Merge routes -- -from ._merge_routes import get_merge_status, get_merge_conflicts, resolve_merge_conflicts, abort_merge, continue_merge # noqa: E501, F401 +from ._config_routes import create_git_config, delete_git_config, get_git_configs, test_git_config, update_git_config # noqa: F401 # -- Environment routes -- -from ._environment_routes import get_environments # noqa: E501, F401 +from ._environment_routes import get_environments # noqa: F401 + +# -- Gitea routes -- +from ._gitea_routes import create_gitea_repository, create_remote_repository, delete_gitea_repository, list_gitea_repositories # noqa: F401 +from ._helpers import _resolve_dashboard_id_from_ref, _resolve_dashboard_id_from_ref_async, _resolve_repo_key_from_ref # noqa: F401 — re-exported for test monkeypatch compatibility + +# -- Merge routes -- +from ._merge_routes import abort_merge, continue_merge, get_merge_conflicts, get_merge_status, resolve_merge_conflicts # noqa: F401 + +# -- Repo lifecycle routes (sync, promote, deploy) -- +from ._repo_lifecycle_routes import deploy_dashboard, promote_dashboard, sync_dashboard # noqa: F401 + +# -- Repo operations routes (commit, push, pull, status, diff, history, generate-message) -- +from ._repo_operations_routes import commit_changes, generate_commit_message, get_history, get_repository_diff, get_repository_status, get_repository_status_batch, pull_changes, push_changes # noqa: F401 + +# -- Repo routes (core) -- +from ._repo_routes import checkout_branch, create_branch, delete_repository, get_branches, get_repository_binding, init_repository # noqa: F401 # #endregion GitPackage diff --git a/backend/src/api/routes/git/_config_routes.py b/backend/src/api/routes/git/_config_routes.py index c89bff2a..a19f3d2b 100644 --- a/backend/src/api/routes/git/_config_routes.py +++ b/backend/src/api/routes/git/_config_routes.py @@ -2,30 +2,27 @@ # @BRIEF FastAPI endpoints for Git server configuration CRUD and connection testing. # @LAYER: API -from typing import List from fastapi import Depends, HTTPException from sqlalchemy.orm import Session -from src.core.database import get_db -from src.core.logger import belief_scope, logger -from src.dependencies import has_permission -from src.models.git import GitServerConfig - from src.api.routes.git_schemas import ( GitServerConfigCreate, GitServerConfigSchema, GitServerConfigUpdate, ) +from src.core.database import get_db +from src.core.logger import belief_scope +from src.dependencies import has_permission +from src.models.git import GitServerConfig -from ._router import router -from ._helpers import _get_git_config_or_404 from ._deps import get_git_service +from ._router import router # #region get_git_configs [C:2] [TYPE Function] # @BRIEF List all configured Git servers. -@router.get("/config", response_model=List[GitServerConfigSchema]) +@router.get("/config", response_model=list[GitServerConfigSchema]) async def get_git_configs( db: Session = Depends(get_db), _=Depends(has_permission("git_config", "READ")), diff --git a/backend/src/api/routes/git/_environment_routes.py b/backend/src/api/routes/git/_environment_routes.py index 0ccc33ba..efd9a6fa 100644 --- a/backend/src/api/routes/git/_environment_routes.py +++ b/backend/src/api/routes/git/_environment_routes.py @@ -2,21 +2,19 @@ # @BRIEF FastAPI endpoint for listing deployment environments. # @LAYER: API -from typing import List from fastapi import Depends +from src.api.routes.git_schemas import DeploymentEnvironmentSchema from src.core.logger import belief_scope from src.dependencies import get_config_manager, has_permission -from src.api.routes.git_schemas import DeploymentEnvironmentSchema - from ._router import router # #region get_environments [C:2] [TYPE Function] # @BRIEF List all deployment environments. -@router.get("/environments", response_model=List[DeploymentEnvironmentSchema]) +@router.get("/environments", response_model=list[DeploymentEnvironmentSchema]) async def get_environments( config_manager=Depends(get_config_manager), _=Depends(has_permission("environments", "READ")), diff --git a/backend/src/api/routes/git/_gitea_routes.py b/backend/src/api/routes/git/_gitea_routes.py index 52be8d53..af756cca 100644 --- a/backend/src/api/routes/git/_gitea_routes.py +++ b/backend/src/api/routes/git/_gitea_routes.py @@ -2,31 +2,29 @@ # @BRIEF FastAPI endpoints for Gitea-specific repository operations. # @LAYER: API -from typing import List from fastapi import Depends, HTTPException from sqlalchemy.orm import Session -from src.core.database import get_db -from src.core.logger import belief_scope -from src.dependencies import has_permission -from src.models.git import GitProvider - from src.api.routes.git_schemas import ( GiteaRepoCreateRequest, GiteaRepoSchema, RemoteRepoCreateRequest, RemoteRepoSchema, ) +from src.core.database import get_db +from src.core.logger import belief_scope +from src.dependencies import has_permission +from src.models.git import GitProvider -from ._router import router -from ._helpers import _get_git_config_or_404 from ._deps import get_git_service +from ._helpers import _get_git_config_or_404 +from ._router import router # #region list_gitea_repositories [C:2] [TYPE Function] # @BRIEF List repositories in Gitea for a saved Gitea config. -@router.get("/config/{config_id}/gitea/repos", response_model=List[GiteaRepoSchema]) +@router.get("/config/{config_id}/gitea/repos", response_model=list[GiteaRepoSchema]) async def list_gitea_repositories( config_id: str, db: Session = Depends(get_db), diff --git a/backend/src/api/routes/git/_helpers.py b/backend/src/api/routes/git/_helpers.py index 6c1109e8..79c80745 100644 --- a/backend/src/api/routes/git/_helpers.py +++ b/backend/src/api/routes/git/_helpers.py @@ -6,19 +6,17 @@ # @RELATION USES -> [UserDashboardPreference] import os -from typing import Optional from fastapi import HTTPException from sqlalchemy.orm import Session from src.core.logger import logger from src.core.superset_client import SupersetClient -from src.dependencies import get_config_manager from src.models.auth import User from src.models.git import GitServerConfig from src.models.profile import UserDashboardPreference -from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH +from ._deps import get_git_service # #region _build_no_repo_status_payload [C:1] [TYPE Function] @@ -49,7 +47,7 @@ def _build_no_repo_status_payload() -> dict: # @POST: Raises HTTPException(500) with route-specific context. def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None: logger.error(f"[{route_name}][Coherence:Failed] {error}") - raise HTTPException(status_code=500, detail=f"{route_name} failed: {str(error)}") + raise HTTPException(status_code=500, detail=f"{route_name} failed: {error!s}") # #endregion _handle_unexpected_git_route_error @@ -95,7 +93,7 @@ def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig: def _find_dashboard_id_by_slug( client: SupersetClient, dashboard_slug: str, -) -> Optional[int]: +) -> int | None: query_variants = [ { "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], @@ -127,7 +125,7 @@ def _find_dashboard_id_by_slug( def _resolve_dashboard_id_from_ref( dashboard_ref: str, config_manager, - env_id: Optional[str] = None, + env_id: str | None = None, ) -> int: normalized_ref = str(dashboard_ref or "").strip() if not normalized_ref: @@ -161,7 +159,7 @@ def _resolve_dashboard_id_from_ref( async def _find_dashboard_id_by_slug_async( client: "AsyncSupersetClient", dashboard_slug: str, -) -> Optional[int]: +) -> int | None: query_variants = [ { "filters": [{"col": "slug", "opr": "eq", "value": dashboard_slug}], @@ -193,7 +191,7 @@ async def _find_dashboard_id_by_slug_async( async def _resolve_dashboard_id_from_ref_async( dashboard_ref: str, config_manager, - env_id: Optional[str] = None, + env_id: str | None = None, ) -> int: normalized_ref = str(dashboard_ref or "").strip() if not normalized_ref: @@ -234,7 +232,7 @@ def _resolve_repo_key_from_ref( dashboard_ref: str, dashboard_id: int, config_manager, - env_id: Optional[str] = None, + env_id: str | None = None, ) -> str: normalized_ref = str(dashboard_ref or "").strip() if normalized_ref and not normalized_ref.isdigit(): @@ -261,7 +259,7 @@ def _resolve_repo_key_from_ref( # #region _sanitize_optional_identity_value [C:1] [TYPE Function] # @BRIEF Normalize optional identity value into trimmed string or None. -def _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]: +def _sanitize_optional_identity_value(value: str | None) -> str | None: normalized = str(value or "").strip() if not normalized: return None @@ -273,8 +271,8 @@ def _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]: # @BRIEF Resolve configured Git username/email from current user's profile preferences. def _resolve_current_user_git_identity( db: Session, - current_user: Optional[User], -) -> Optional[tuple[str, str]]: + current_user: User | None, +) -> tuple[str, str] | None: if db is None or not hasattr(db, "query"): return None @@ -315,7 +313,7 @@ def _resolve_current_user_git_identity( def _apply_git_identity_from_profile( dashboard_id: int, db: Session, - current_user: Optional[User], + current_user: User | None, ) -> None: identity = _resolve_current_user_git_identity(db, current_user) if not identity: diff --git a/backend/src/api/routes/git/_merge_routes.py b/backend/src/api/routes/git/_merge_routes.py index ec6f95e0..a15bfa2d 100644 --- a/backend/src/api/routes/git/_merge_routes.py +++ b/backend/src/api/routes/git/_merge_routes.py @@ -2,25 +2,23 @@ # @BRIEF FastAPI endpoints for merge operations (status, conflicts, resolve, abort, continue). # @LAYER: API -from typing import List, Optional from fastapi import Depends, HTTPException -from src.core.logger import belief_scope -from src.dependencies import get_config_manager, has_permission - from src.api.routes.git_schemas import ( MergeConflictFileSchema, MergeContinueRequest, MergeResolveRequest, MergeStatusSchema, ) +from src.core.logger import belief_scope +from src.dependencies import get_config_manager, has_permission -from ._router import router +from ._deps import get_git_service from ._helpers import ( _handle_unexpected_git_route_error, ) -from ._deps import get_git_service +from ._router import router # #region get_merge_status [C:2] [TYPE Function] @@ -30,7 +28,7 @@ from ._deps import get_git_service ) async def get_merge_status( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -54,11 +52,11 @@ async def get_merge_status( # @BRIEF Return conflicted files with mine/theirs previews for web conflict resolver. @router.get( "/repositories/{dashboard_ref}/merge/conflicts", - response_model=List[MergeConflictFileSchema], + response_model=list[MergeConflictFileSchema], ) async def get_merge_conflicts( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -85,7 +83,7 @@ async def get_merge_conflicts( async def resolve_merge_conflicts( dashboard_ref: str, resolve_data: MergeResolveRequest, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -114,7 +112,7 @@ async def resolve_merge_conflicts( @router.post("/repositories/{dashboard_ref}/merge/abort") async def abort_merge( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -141,7 +139,7 @@ async def abort_merge( async def continue_merge( dashboard_ref: str, continue_data: MergeContinueRequest, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): diff --git a/backend/src/api/routes/git/_repo_lifecycle_routes.py b/backend/src/api/routes/git/_repo_lifecycle_routes.py index 5dcf253a..2f7a41fe 100644 --- a/backend/src/api/routes/git/_repo_lifecycle_routes.py +++ b/backend/src/api/routes/git/_repo_lifecycle_routes.py @@ -2,31 +2,28 @@ # @BRIEF FastAPI endpoints for Git lifecycle operations (sync, promote, deploy). # @LAYER: API -import typing -from typing import Optional from fastapi import Depends, HTTPException from sqlalchemy.orm import Session -from src.core.database import get_db -from src.core.logger import belief_scope, logger -from src.dependencies import get_config_manager, get_current_user, has_permission -from src.models.auth import User -from src.models.git import GitProvider, GitRepository - from src.api.routes.git_schemas import ( DeployRequest, PromoteRequest, PromoteResponse, ) +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.git import GitProvider, GitRepository -from ._router import router +from ._deps import get_git_service from ._helpers import ( _apply_git_identity_from_profile, _get_git_config_or_404, _handle_unexpected_git_route_error, ) -from ._deps import get_git_service +from ._router import router # #region sync_dashboard [C:3] [TYPE Function] @@ -35,8 +32,8 @@ from ._deps import get_git_service @router.post("/repositories/{dashboard_ref}/sync") async def sync_dashboard( dashboard_ref: str, - env_id: Optional[str] = None, - source_env_id: typing.Optional[str] = None, + env_id: str | None = None, + source_env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -71,7 +68,7 @@ async def sync_dashboard( async def promote_dashboard( dashboard_ref: str, payload: PromoteRequest, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), current_user: User = Depends(get_current_user), @@ -194,7 +191,7 @@ async def promote_dashboard( async def deploy_dashboard( dashboard_ref: str, deploy_data: DeployRequest, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): diff --git a/backend/src/api/routes/git/_repo_operations_routes.py b/backend/src/api/routes/git/_repo_operations_routes.py index 661b3cc8..9998d13e 100644 --- a/backend/src/api/routes/git/_repo_operations_routes.py +++ b/backend/src/api/routes/git/_repo_operations_routes.py @@ -2,32 +2,30 @@ # @BRIEF FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history). # @LAYER: API -from typing import List, Optional from fastapi import Depends, HTTPException from sqlalchemy.orm import Session -from src.core.database import get_db -from src.core.logger import belief_scope, logger -from src.dependencies import get_config_manager, get_current_user, has_permission -from src.models.auth import User -from src.models.git import GitRepository, GitServerConfig - from src.api.routes.git_schemas import ( CommitCreate, CommitSchema, RepoStatusBatchRequest, RepoStatusBatchResponse, ) +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.git import GitRepository, GitServerConfig -from ._router import router +from ._deps import MAX_REPOSITORY_STATUS_BATCH, get_git_service from ._helpers import ( _apply_git_identity_from_profile, _build_no_repo_status_payload, _handle_unexpected_git_route_error, _resolve_repository_status, ) -from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH +from ._router import router # #region commit_changes [C:2] [TYPE Function] @@ -36,7 +34,7 @@ from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH async def commit_changes( dashboard_ref: str, commit_data: CommitCreate, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), current_user: User = Depends(get_current_user), @@ -67,7 +65,7 @@ async def commit_changes( @router.post("/repositories/{dashboard_ref}/push") async def push_changes( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -94,7 +92,7 @@ async def push_changes( @router.post("/repositories/{dashboard_ref}/pull") async def pull_changes( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), current_user: User = Depends(get_current_user), @@ -155,7 +153,7 @@ async def pull_changes( @router.get("/repositories/{dashboard_ref}/status") async def get_repository_status( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -218,9 +216,9 @@ async def get_repository_status_batch( @router.get("/repositories/{dashboard_ref}/diff") async def get_repository_diff( dashboard_ref: str, - file_path: Optional[str] = None, + file_path: str | None = None, staged: bool = False, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -242,11 +240,11 @@ async def get_repository_diff( # #region get_history [C:2] [TYPE Function] # @BRIEF View commit history for a dashboard's repository. -@router.get("/repositories/{dashboard_ref}/history", response_model=List[CommitSchema]) +@router.get("/repositories/{dashboard_ref}/history", response_model=list[CommitSchema]) async def get_history( dashboard_ref: str, limit: int = 50, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -272,7 +270,7 @@ async def get_history( @router.post("/repositories/{dashboard_ref}/generate-message") async def generate_commit_message( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), _=Depends(has_permission("plugin:git", "EXECUTE")), @@ -295,14 +293,14 @@ async def generate_commit_message( history_objs = _gs.get_commit_history(dashboard_id, limit=5) history = [h.message for h in history_objs if hasattr(h, "message")] - from ...services.llm_provider import LLMProviderService - from ...plugins.llm_analysis.service import LLMClient from ...plugins.llm_analysis.models import LLMProviderType + from ...plugins.llm_analysis.service import LLMClient from ...services.llm_prompt_templates import ( DEFAULT_LLM_PROMPTS, normalize_llm_settings, resolve_bound_provider_id, ) + from ...services.llm_provider import LLMProviderService llm_service = LLMProviderService(db) providers = llm_service.get_all_providers() diff --git a/backend/src/api/routes/git/_repo_routes.py b/backend/src/api/routes/git/_repo_routes.py index c095d965..1eb84b30 100644 --- a/backend/src/api/routes/git/_repo_routes.py +++ b/backend/src/api/routes/git/_repo_routes.py @@ -2,17 +2,10 @@ # @BRIEF FastAPI endpoints for core Git repository operations (init, binding, branches, checkout). # @LAYER: API -from typing import List, Optional from fastapi import Depends, HTTPException from sqlalchemy.orm import Session -from src.core.database import get_db -from src.core.logger import belief_scope, logger -from src.dependencies import get_config_manager, get_current_user, has_permission -from src.models.auth import User -from src.models.git import GitRepository, GitServerConfig - from src.api.routes.git_schemas import ( BranchCheckout, BranchCreate, @@ -20,14 +13,19 @@ from src.api.routes.git_schemas import ( RepoInitRequest, RepositoryBindingSchema, ) +from src.core.database import get_db +from src.core.logger import belief_scope, logger +from src.dependencies import get_config_manager, get_current_user, has_permission +from src.models.auth import User +from src.models.git import GitRepository, GitServerConfig -from ._router import router +from ._deps import get_git_service from ._helpers import ( _apply_git_identity_from_profile, _get_git_config_or_404, _handle_unexpected_git_route_error, ) -from ._deps import get_git_service +from ._router import router # #region init_repository [C:3] [TYPE Function] @@ -37,7 +35,7 @@ from ._deps import get_git_service async def init_repository( dashboard_ref: str, init_data: RepoInitRequest, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), _=Depends(has_permission("plugin:git", "EXECUTE")), @@ -115,7 +113,7 @@ async def init_repository( @router.get("/repositories/{dashboard_ref}", response_model=RepositoryBindingSchema) async def get_repository_binding( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), _=Depends(has_permission("plugin:git", "EXECUTE")), @@ -156,7 +154,7 @@ async def get_repository_binding( @router.delete("/repositories/{dashboard_ref}") async def delete_repository( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -179,10 +177,10 @@ async def delete_repository( # #region get_branches [C:2] [TYPE Function] # @BRIEF List all branches for a dashboard's repository. -@router.get("/repositories/{dashboard_ref}/branches", response_model=List[BranchSchema]) +@router.get("/repositories/{dashboard_ref}/branches", response_model=list[BranchSchema]) async def get_branches( dashboard_ref: str, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): @@ -208,7 +206,7 @@ async def get_branches( async def create_branch( dashboard_ref: str, branch_data: BranchCreate, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), db: Session = Depends(get_db), current_user: User = Depends(get_current_user), @@ -240,7 +238,7 @@ async def create_branch( async def checkout_branch( dashboard_ref: str, checkout_data: BranchCheckout, - env_id: Optional[str] = None, + env_id: str | None = None, config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:git", "EXECUTE")), ): diff --git a/backend/src/api/routes/git_schemas.py b/backend/src/api/routes/git_schemas.py index 51800c57..e1bc5a7b 100644 --- a/backend/src/api/routes/git_schemas.py +++ b/backend/src/api/routes/git_schemas.py @@ -6,11 +6,14 @@ # # @INVARIANT: All schemas must be compatible with the FastAPI router. -from pydantic import BaseModel, Field -from typing import Any, Dict, List, Optional from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field + from src.models.git import GitProvider, GitStatus, SyncStatus + # #region GitServerConfigBase [C:1] [TYPE Class] # @BRIEF Base schema for Git server configuration attributes. class GitServerConfigBase(BaseModel): @@ -19,26 +22,26 @@ class GitServerConfigBase(BaseModel): url: str = Field(..., description="Server base URL") pat: str = Field(..., description="Personal Access Token") pat: str = Field(..., description="Personal Access Token") - default_repository: Optional[str] = Field(None, description="Default repository path (org/repo)") - default_branch: Optional[str] = Field("main", description="Default branch logic/name") + default_repository: str | None = Field(None, description="Default repository path (org/repo)") + default_branch: str | None = Field("main", description="Default branch logic/name") # #endregion GitServerConfigBase # #region GitServerConfigUpdate [TYPE Class] # @BRIEF Schema for updating an existing Git server configuration. class GitServerConfigUpdate(BaseModel): - name: Optional[str] = Field(None, description="Display name for the Git server") - provider: Optional[GitProvider] = Field(None, description="Git provider (GITHUB, GITLAB, GITEA)") - url: Optional[str] = Field(None, description="Server base URL") - pat: Optional[str] = Field(None, description="Personal Access Token") - default_repository: Optional[str] = Field(None, description="Default repository path (org/repo)") - default_branch: Optional[str] = Field(None, description="Default branch logic/name") + name: str | None = Field(None, description="Display name for the Git server") + provider: GitProvider | None = Field(None, description="Git provider (GITHUB, GITLAB, GITEA)") + url: str | None = Field(None, description="Server base URL") + pat: str | None = Field(None, description="Personal Access Token") + default_repository: str | None = Field(None, description="Default repository path (org/repo)") + default_branch: str | None = Field(None, description="Default branch logic/name") # #endregion GitServerConfigUpdate # #region GitServerConfigCreate [TYPE Class] # @BRIEF Schema for creating a new Git server configuration. class GitServerConfigCreate(GitServerConfigBase): """Schema for creating a new Git server configuration.""" - config_id: Optional[str] = Field(None, description="Optional config ID, useful for testing an existing config without sending its full PAT") + config_id: str | None = Field(None, description="Optional config ID, useful for testing an existing config without sending its full PAT") # #endregion GitServerConfigCreate # #region GitServerConfigSchema [TYPE Class] @@ -88,7 +91,7 @@ class CommitSchema(BaseModel): email: str timestamp: datetime message: str - files_changed: List[str] + files_changed: list[str] # #endregion CommitSchema # #region BranchCreate [TYPE Class] @@ -111,7 +114,7 @@ class BranchCheckout(BaseModel): class CommitCreate(BaseModel): """Schema for staging and committing changes.""" message: str - files: List[str] + files: list[str] # #endregion CommitCreate # #region ConflictResolution [TYPE Class] @@ -120,7 +123,7 @@ class ConflictResolution(BaseModel): """Schema for resolving merge conflicts.""" file_path: str resolution: str = Field(pattern="^(mine|theirs|manual)$") - content: Optional[str] = None + content: str | None = None # #endregion ConflictResolution @@ -131,8 +134,8 @@ class MergeStatusSchema(BaseModel): repository_path: str git_dir: str current_branch: str - merge_head: Optional[str] = None - merge_message_preview: Optional[str] = None + merge_head: str | None = None + merge_message_preview: str | None = None conflicts_count: int = 0 # #endregion MergeStatusSchema @@ -141,22 +144,22 @@ class MergeStatusSchema(BaseModel): # @BRIEF Schema describing one conflicted file with optional side snapshots. class MergeConflictFileSchema(BaseModel): file_path: str - mine: Optional[str] = None - theirs: Optional[str] = None + mine: str | None = None + theirs: str | None = None # #endregion MergeConflictFileSchema # #region MergeResolveRequest [TYPE Class] # @BRIEF Request schema for resolving one or multiple merge conflicts. class MergeResolveRequest(BaseModel): - resolutions: List[ConflictResolution] = Field(default_factory=list) + resolutions: list[ConflictResolution] = Field(default_factory=list) # #endregion MergeResolveRequest # #region MergeContinueRequest [TYPE Class] # @BRIEF Request schema for finishing merge with optional explicit commit message. class MergeContinueRequest(BaseModel): - message: Optional[str] = None + message: str | None = None # #endregion MergeContinueRequest # #region DeploymentEnvironmentSchema [TYPE Class] @@ -201,14 +204,14 @@ class RepositoryBindingSchema(BaseModel): # #region RepoStatusBatchRequest [TYPE Class] # @BRIEF Schema for requesting repository statuses for multiple dashboards in a single call. class RepoStatusBatchRequest(BaseModel): - dashboard_ids: List[int] = Field(default_factory=list, description="Dashboard IDs to resolve repository statuses for") + dashboard_ids: list[int] = Field(default_factory=list, description="Dashboard IDs to resolve repository statuses for") # #endregion RepoStatusBatchRequest # #region RepoStatusBatchResponse [TYPE Class] # @BRIEF Schema for returning repository statuses keyed by dashboard ID. class RepoStatusBatchResponse(BaseModel): - statuses: Dict[str, Dict[str, Any]] + statuses: dict[str, dict[str, Any]] # #endregion RepoStatusBatchResponse @@ -218,10 +221,10 @@ class GiteaRepoSchema(BaseModel): name: str full_name: str private: bool = False - clone_url: Optional[str] = None - html_url: Optional[str] = None - ssh_url: Optional[str] = None - default_branch: Optional[str] = None + clone_url: str | None = None + html_url: str | None = None + ssh_url: str | None = None + default_branch: str | None = None # #endregion GiteaRepoSchema @@ -230,9 +233,9 @@ class GiteaRepoSchema(BaseModel): class GiteaRepoCreateRequest(BaseModel): name: str = Field(..., min_length=1, max_length=255) private: bool = True - description: Optional[str] = None + description: str | None = None auto_init: bool = True - default_branch: Optional[str] = "main" + default_branch: str | None = "main" # #endregion GiteaRepoCreateRequest @@ -243,10 +246,10 @@ class RemoteRepoSchema(BaseModel): name: str full_name: str private: bool = False - clone_url: Optional[str] = None - html_url: Optional[str] = None - ssh_url: Optional[str] = None - default_branch: Optional[str] = None + clone_url: str | None = None + html_url: str | None = None + ssh_url: str | None = None + default_branch: str | None = None # #endregion RemoteRepoSchema @@ -255,9 +258,9 @@ class RemoteRepoSchema(BaseModel): class RemoteRepoCreateRequest(BaseModel): name: str = Field(..., min_length=1, max_length=255) private: bool = True - description: Optional[str] = None + description: str | None = None auto_init: bool = True - default_branch: Optional[str] = "main" + default_branch: str | None = "main" # #endregion RemoteRepoCreateRequest @@ -267,9 +270,9 @@ class PromoteRequest(BaseModel): from_branch: str = Field(..., min_length=1, max_length=255) to_branch: str = Field(..., min_length=1, max_length=255) mode: str = Field(default="mr", pattern="^(mr|direct)$") - title: Optional[str] = None - description: Optional[str] = None - reason: Optional[str] = None + title: str | None = None + description: str | None = None + reason: str | None = None draft: bool = False remove_source_branch: bool = False # #endregion PromoteRequest @@ -282,8 +285,8 @@ class PromoteResponse(BaseModel): from_branch: str to_branch: str status: str - url: Optional[str] = None - reference_id: Optional[str] = None + url: str | None = None + reference_id: str | None = None policy_violation: bool = False # #endregion PromoteResponse diff --git a/backend/src/api/routes/health.py b/backend/src/api/routes/health.py index 9cea35e4..f2fcbbfe 100644 --- a/backend/src/api/routes/health.py +++ b/backend/src/api/routes/health.py @@ -3,13 +3,14 @@ # @LAYER: UI/API # @RELATION DEPENDS_ON -> health_service -from fastapi import APIRouter, Depends, Query, HTTPException, status -from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy.orm import Session + from ...core.database import get_db -from ...services.health_service import HealthService +from ...dependencies import get_config_manager, get_task_manager, has_permission from ...schemas.health import HealthSummaryResponse -from ...dependencies import has_permission, get_config_manager, get_task_manager +from ...services.health_service import HealthService router = APIRouter(prefix="/api/health", tags=["Health"]) @@ -20,7 +21,7 @@ router = APIRouter(prefix="/api/health", tags=["Health"]) # @RELATION CALLS -> backend.src.services.health_service.HealthService @router.get("/summary", response_model=HealthSummaryResponse) async def get_health_summary( - environment_id: Optional[str] = Query(None), + environment_id: str | None = Query(None), db: Session = Depends(get_db), config_manager = Depends(get_config_manager), _ = Depends(has_permission("plugin:migration", "READ")) diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 44c70d97..2c3bf091 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -6,25 +6,26 @@ # @RELATION DEPENDS_ON -> [get_current_user] # @RELATION DEPENDS_ON -> [get_db] + from fastapi import APIRouter, Depends, HTTPException, status -from typing import List, Optional from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from ...core.database import get_db from ...core.logger import logger -from ...schemas.auth import User from ...dependencies import get_current_user as get_current_active_user from ...plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType +from ...schemas.auth import User from ...services.llm_provider import LLMProviderService -from ...core.database import get_db -from sqlalchemy.orm import Session # #region FetchModelsRequest [C:1] [TYPE Class] # @BRIEF Pydantic request model for the fetch-models endpoint. class FetchModelsRequest(BaseModel): - base_url: Optional[str] = Field(None, description="LLM provider base URL") - provider_type: Optional[str] = Field(None, description="Provider type (openai, anthropic, etc)") - api_key: Optional[str] = Field(None, description="Direct API key (takes precedence over provider_id)") - provider_id: Optional[str] = Field(None, description="Saved provider ID to look up stored API key") + base_url: str | None = Field(None, description="LLM provider base URL") + provider_type: str | None = Field(None, description="Provider type (openai, anthropic, etc)") + api_key: str | None = Field(None, description="Direct API key (takes precedence over provider_id)") + provider_id: str | None = Field(None, description="Saved provider ID to look up stored API key") # #endregion FetchModelsRequest @@ -40,7 +41,7 @@ router = APIRouter(tags=["LLM"]) # @PRE: value can be None. # @POST: Returns True only for non-placeholder key. # @RELATION BINDS_TO -> [LlmRoutes] -def _is_valid_runtime_api_key(value: Optional[str]) -> bool: +def _is_valid_runtime_api_key(value: str | None) -> bool: key = (value or "").strip() if not key: return False @@ -58,7 +59,7 @@ def _is_valid_runtime_api_key(value: Optional[str]) -> bool: # @POST: Returns list of LLMProviderConfig. # @RELATION CALLS -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMProviderConfig] -@router.get("/providers", response_model=List[LLMProviderConfig]) +@router.get("/providers", response_model=list[LLMProviderConfig]) async def get_providers( current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db) ): @@ -100,8 +101,8 @@ async def fetch_models( current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db), ): - from ...plugins.llm_analysis.service import LLMClient from ...plugins.llm_analysis.models import LLMProviderType + from ...plugins.llm_analysis.service import LLMClient base_url = (payload.base_url or "").strip() provider_type_str = (payload.provider_type or "").strip() diff --git a/backend/src/api/routes/mappings.py b/backend/src/api/routes/mappings.py index 503328a6..2af2a357 100644 --- a/backend/src/api/routes/mappings.py +++ b/backend/src/api/routes/mappings.py @@ -8,14 +8,15 @@ # # @INVARIANT: Mappings are persisted in the SQLite database. + from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel from sqlalchemy.orm import Session -from typing import List, Optional + +from ...core.database import get_db from ...core.logger import belief_scope from ...dependencies import get_config_manager, has_permission -from ...core.database import get_db from ...models.mapping import DatabaseMapping -from pydantic import BaseModel router = APIRouter(tags=["mappings"]) @@ -27,7 +28,7 @@ class MappingCreate(BaseModel): target_db_uuid: str source_db_name: str target_db_name: str - engine: Optional[str] = None + engine: str | None = None # #endregion MappingCreate # #region MappingResponse [TYPE DataClass] @@ -39,7 +40,7 @@ class MappingResponse(BaseModel): target_db_uuid: str source_db_name: str target_db_name: str - engine: Optional[str] = None + engine: str | None = None class Config: from_attributes = True @@ -55,10 +56,10 @@ class SuggestRequest(BaseModel): # @BRIEF List all saved database mappings. # @PRE: db session is injected. # @POST: Returns filtered list of DatabaseMapping records. -@router.get("", response_model=List[MappingResponse]) +@router.get("", response_model=list[MappingResponse]) async def get_mappings( - source_env_id: Optional[str] = None, - target_env_id: Optional[str] = None, + source_env_id: str | None = None, + target_env_id: str | None = None, db: Session = Depends(get_db), _ = Depends(has_permission("plugin:mapper", "EXECUTE")) ): @@ -88,7 +89,7 @@ async def create_mapping( DatabaseMapping.target_env_id == mapping.target_env_id, DatabaseMapping.source_db_uuid == mapping.source_db_uuid ).first() - + if existing: existing.target_db_uuid = mapping.target_db_uuid existing.target_db_name = mapping.target_db_name @@ -96,7 +97,7 @@ async def create_mapping( db.commit() db.refresh(existing) return existing - + new_mapping = DatabaseMapping(**mapping.dict()) db.add(new_mapping) db.commit() diff --git a/backend/src/api/routes/migration.py b/backend/src/api/routes/migration.py index e157318b..fbf40554 100644 --- a/backend/src/api/routes/migration.py +++ b/backend/src/api/routes/migration.py @@ -21,16 +21,18 @@ # @TEST_EDGE: [external_fail] ->[HTTP_500] # @TEST_INVARIANT: [EnvironmentValidationBeforeAction] -> VERIFIED_BY: [invalid_environment, valid_execution] +from typing import Any, cast + from fastapi import APIRouter, Depends, HTTPException, Query -from typing import List, Dict, Any, Optional, cast from sqlalchemy.orm import Session -from ...dependencies import get_config_manager, get_task_manager, has_permission + from ...core.database import get_db -from ...models.dashboard import DashboardMetadata, DashboardSelection -from ...core.superset_client import SupersetClient -from ...core.logger import logger, belief_scope -from ...core.migration.dry_run_orchestrator import MigrationDryRunService +from ...core.logger import belief_scope, logger from ...core.mapping_service import IdMappingService +from ...core.migration.dry_run_orchestrator import MigrationDryRunService +from ...core.superset_client import SupersetClient +from ...dependencies import get_config_manager, get_task_manager, has_permission +from ...models.dashboard import DashboardMetadata, DashboardSelection from ...models.mapping import ResourceMapping logger = cast(Any, logger) @@ -45,7 +47,7 @@ router = APIRouter(prefix="/api", tags=["migration"]) # @SIDE_EFFECT: Reads environment configuration and performs remote Superset metadata retrieval over network. # @DATA_CONTRACT: Input[str env_id] -> Output[List[DashboardMetadata]] # @RELATION CALLS -> [SupersetClient.get_dashboards_summary] -@router.get("/environments/{env_id}/dashboards", response_model=List[DashboardMetadata]) +@router.get("/environments/{env_id}/dashboards", response_model=list[DashboardMetadata]) async def get_dashboards( env_id: str, config_manager=Depends(get_config_manager), @@ -125,7 +127,7 @@ async def execute_migration( except Exception as e: logger.explore(f"Task creation failed: {e}") raise HTTPException( - status_code=500, detail=f"Failed to create migration task: {str(e)}" + status_code=500, detail=f"Failed to create migration task: {e!s}" ) @@ -141,7 +143,7 @@ async def execute_migration( # @RELATION DEPENDS_ON -> [DashboardSelection] # @RELATION DEPENDS_ON -> [MigrationDryRunService] # @INVARIANT: Dry-run flow remains read-only and rejects identical source/target environments before service execution. -@router.post("/migration/dry-run", response_model=Dict[str, Any]) +@router.post("/migration/dry-run", response_model=dict[str, Any]) async def dry_run_migration( selection: DashboardSelection, config_manager=Depends(get_config_manager), @@ -205,7 +207,7 @@ async def dry_run_migration( # @SIDE_EFFECT: Reads configuration from config manager. # @DATA_CONTRACT: Input[None] -> Output[Dict[str, str]] # @RELATION DEPENDS_ON -> [AppDependencies] -@router.get("/migration/settings", response_model=Dict[str, str]) +@router.get("/migration/settings", response_model=dict[str, str]) async def get_migration_settings( config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:migration", "READ")), @@ -226,9 +228,9 @@ async def get_migration_settings( # @SIDE_EFFECT: Mutates configuration and writes persisted config through config manager. # @DATA_CONTRACT: Input[Dict[str, str]] -> Output[Dict[str, str]] # @RELATION DEPENDS_ON -> [AppDependencies] -@router.put("/migration/settings", response_model=Dict[str, str]) +@router.put("/migration/settings", response_model=dict[str, str]) async def update_migration_settings( - payload: Dict[str, str], + payload: dict[str, str], config_manager=Depends(get_config_manager), _=Depends(has_permission("plugin:migration", "WRITE")), ): @@ -257,13 +259,13 @@ async def update_migration_settings( # @SIDE_EFFECT: Executes database read queries against ResourceMapping table. # @DATA_CONTRACT: Input[QueryParams] -> Output[Dict[str, Any]] # @RELATION DEPENDS_ON -> [ResourceMapping] -@router.get("/migration/mappings-data", response_model=Dict[str, Any]) +@router.get("/migration/mappings-data", response_model=dict[str, Any]) async def get_resource_mappings( skip: int = Query(0, ge=0), limit: int = Query(50, ge=1, le=500), - search: Optional[str] = Query(None, description="Search by resource name or UUID"), - env_id: Optional[str] = Query(None, description="Filter by environment ID"), - resource_type: Optional[str] = Query(None, description="Filter by resource type"), + search: str | None = Query(None, description="Search by resource name or UUID"), + env_id: str | None = Query(None, description="Filter by environment ID"), + resource_type: str | None = Query(None, description="Filter by resource type"), db: Session = Depends(get_db), _=Depends(has_permission("plugin:migration", "READ")), ): @@ -330,7 +332,7 @@ async def get_resource_mappings( # @DATA_CONTRACT: Input[None] -> Output[Dict[str, Any]] # @RELATION DEPENDS_ON -> [IdMappingService] # @RELATION CALLS -> [sync_environment] -@router.post("/migration/sync-now", response_model=Dict[str, Any]) +@router.post("/migration/sync-now", response_model=dict[str, Any]) async def trigger_sync_now( config_manager=Depends(get_config_manager), db: Session = Depends(get_db), diff --git a/backend/src/api/routes/plugins.py b/backend/src/api/routes/plugins.py index f17c6972..8f5c0107 100755 --- a/backend/src/api/routes/plugins.py +++ b/backend/src/api/routes/plugins.py @@ -4,12 +4,12 @@ # @RELATION DEPENDS_ON -> [PluginConfig] # @RELATION DEPENDS_ON -> [get_plugin_loader] # @RELATION BINDS_TO -> [API_Routes] -from typing import List + from fastapi import APIRouter, Depends +from ...core.logger import belief_scope from ...core.plugin_base import PluginConfig from ...dependencies import get_plugin_loader, has_permission -from ...core.logger import belief_scope router = APIRouter() @@ -20,7 +20,7 @@ router = APIRouter() # @POST: Returns a list of PluginConfig objects. # @RELATION CALLS -> [get_plugin_loader] # @RELATION DEPENDS_ON -> [PluginConfig] -@router.get("", response_model=List[PluginConfig]) +@router.get("", response_model=list[PluginConfig]) async def list_plugins( plugin_loader=Depends(get_plugin_loader), _=Depends(has_permission("plugins", "READ")), diff --git a/backend/src/api/routes/profile.py b/backend/src/api/routes/profile.py index f1d87b07..5669024e 100644 --- a/backend/src/api/routes/profile.py +++ b/backend/src/api/routes/profile.py @@ -13,13 +13,12 @@ # @UX_FEEDBACK: Stable status/message/warning payloads support profile page feedback. # @UX_RECOVERY: Lookup degradation keeps manual username save path available. -from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from ...core.database import get_db -from ...core.logger import logger, belief_scope +from ...core.logger import belief_scope, logger from ...dependencies import ( get_config_manager, get_current_user, @@ -110,7 +109,7 @@ async def update_preferences( @router.get("/superset-accounts", response_model=SupersetAccountLookupResponse) async def lookup_superset_accounts( environment_id: str = Query(...), - search: Optional[str] = Query(default=None), + search: str | None = Query(default=None), page_index: int = Query(default=0, ge=0), page_size: int = Query(default=20, ge=1, le=100), sort_column: str = Query(default="username"), @@ -144,4 +143,4 @@ async def lookup_superset_accounts( raise HTTPException(status_code=404, detail=str(exc)) from exc # #endregion lookup_superset_accounts -# #endregion ProfileApiModule \ No newline at end of file +# #endregion ProfileApiModule diff --git a/backend/src/api/routes/reports.py b/backend/src/api/routes/reports.py index 1f7fef94..af17e900 100644 --- a/backend/src/api/routes/reports.py +++ b/backend/src/api/routes/reports.py @@ -12,17 +12,16 @@ # @DATA_CONTRACT: [ReportQuery] -> [ReportCollection | ReportDetailView] from datetime import datetime -from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Query, status +from ...core.logger import belief_scope +from ...core.task_manager import TaskManager from ...dependencies import ( + get_clean_release_repository, get_task_manager, has_permission, - get_clean_release_repository, ) -from ...core.task_manager import TaskManager -from ...core.logger import belief_scope from ...models.report import ( ReportCollection, ReportDetailView, @@ -41,7 +40,7 @@ router = APIRouter(prefix="/api/reports", tags=["Reports"]) # @PRE: raw may be None/empty or comma-separated values. # @POST: Returns enum list or raises HTTP 400 with deterministic machine-readable payload. # @RELATION BINDS_TO -> [ReportsRouter] -def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: +def _parse_csv_enum_list(raw: str | None, enum_cls, field_name: str) -> list: with belief_scope("_parse_csv_enum_list"): if raw is None or not raw.strip(): return [] @@ -95,11 +94,11 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: async def list_reports( page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100), - task_types: Optional[str] = Query(None, description="Comma-separated task types"), - statuses: Optional[str] = Query(None, description="Comma-separated statuses"), - time_from: Optional[datetime] = Query(None), - time_to: Optional[datetime] = Query(None), - search: Optional[str] = Query(None, max_length=200), + task_types: str | None = Query(None, description="Comma-separated task types"), + statuses: str | None = Query(None, description="Comma-separated statuses"), + time_from: datetime | None = Query(None), + time_to: datetime | None = Query(None), + search: str | None = Query(None, max_length=200), sort_by: str = Query("updated_at"), sort_order: str = Query("desc"), task_manager: TaskManager = Depends(get_task_manager), diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index 08f31771..f9e8d247 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -9,28 +9,29 @@ # @INVARIANT: All settings changes must be persisted via ConfigManager. # @PUBLIC_API: router + from fastapi import APIRouter, Depends, HTTPException -from typing import List from pydantic import BaseModel -from ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig -from ...models.storage import StorageConfig -from ...dependencies import get_config_manager, has_permission -from ...core.config_manager import ConfigManager -from ...core.logger import logger, belief_scope -from ...core.superset_client import SupersetClient -from ...services.llm_prompt_templates import normalize_llm_settings -from ...models.llm import ValidationPolicy -from ...models.config import AppConfigRecord -from ...schemas.settings import ( - ValidationPolicyCreate, - ValidationPolicyUpdate, - ValidationPolicyResponse, - TranslationScheduleItem, -) -from ...models.translate import TranslationSchedule, TranslationJob -from ...core.database import get_db from sqlalchemy.orm import Session +from ...core.config_manager import ConfigManager +from ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig +from ...core.database import get_db +from ...core.logger import belief_scope, logger +from ...core.superset_client import SupersetClient +from ...dependencies import get_config_manager, has_permission +from ...models.config import AppConfigRecord +from ...models.llm import ValidationPolicy +from ...models.storage import StorageConfig +from ...models.translate import TranslationJob, TranslationSchedule +from ...schemas.settings import ( + TranslationScheduleItem, + ValidationPolicyCreate, + ValidationPolicyResponse, + ValidationPolicyUpdate, +) +from ...services.llm_prompt_templates import normalize_llm_settings + # #region LoggingConfigResponse [C:1] [TYPE Class] [SEMANTICS logging, config, response] # @BRIEF Response model for logging configuration with current task log level. @@ -179,7 +180,7 @@ async def update_storage_settings( # @BRIEF Lists all configured Superset environments. # @PRE: Config manager is available. # @POST: Returns list of environments. -@router.get("/environments", response_model=List[Environment]) +@router.get("/environments", response_model=list[Environment]) async def get_environments( config_manager: ConfigManager = Depends(get_config_manager), _=Depends(has_permission("admin:settings", "READ")), @@ -375,10 +376,10 @@ async def update_logging_config( # #region ConsolidatedSettingsResponse [C:1] [TYPE Class] # @BRIEF Response model for consolidated application settings. class ConsolidatedSettingsResponse(BaseModel): - environments: List[dict] - connections: List[dict] + environments: list[dict] + connections: list[dict] llm: dict - llm_providers: List[dict] + llm_providers: list[dict] logging: dict storage: dict notifications: dict = {} @@ -410,8 +411,8 @@ async def get_consolidated_settings( config = config_manager.get_config() - from ...services.llm_provider import LLMProviderService from ...core.database import SessionLocal + from ...services.llm_provider import LLMProviderService db = SessionLocal() notifications_payload = {} @@ -524,7 +525,7 @@ async def update_consolidated_settings( # #region get_validation_policies [C:2] [TYPE Function] # @BRIEF Lists all validation policies. -@router.get("/automation/policies", response_model=List[ValidationPolicyResponse]) +@router.get("/automation/policies", response_model=list[ValidationPolicyResponse]) async def get_validation_policies( db: Session = Depends(get_db), _=Depends(has_permission("admin:settings", "READ")) ): @@ -603,7 +604,7 @@ async def delete_validation_policy( # #region get_translation_schedules [C:2] [TYPE Function] # @BRIEF Lists all translation schedules joined with translation job names. -@router.get("/automation/translation-schedules", response_model=List[TranslationScheduleItem]) +@router.get("/automation/translation-schedules", response_model=list[TranslationScheduleItem]) async def get_translation_schedules( db: Session = Depends(get_db), _=Depends(has_permission("admin:settings", "READ")), diff --git a/backend/src/api/routes/storage.py b/backend/src/api/routes/storage.py index 5246b0ec..b3e91b38 100644 --- a/backend/src/api/routes/storage.py +++ b/backend/src/api/routes/storage.py @@ -8,13 +8,14 @@ # @INVARIANT: All paths must be validated against path traversal. from pathlib import Path -from fastapi import APIRouter, Depends, UploadFile, File, Form, HTTPException + +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile from fastapi.responses import FileResponse -from typing import List, Optional -from ...models.storage import StoredFile, FileCategory -from ...dependencies import get_plugin_loader, has_permission -from ...plugins.storage.plugin import StoragePlugin + from ...core.logger import belief_scope +from ...dependencies import get_plugin_loader, has_permission +from ...models.storage import FileCategory, StoredFile +from ...plugins.storage.plugin import StoragePlugin router = APIRouter(tags=["storage"]) @@ -25,10 +26,10 @@ router = APIRouter(tags=["storage"]) # @POST: Returns a list of StoredFile objects. # # @RELATION DEPENDS_ON -> [StoragePlugin] -@router.get("/files", response_model=List[StoredFile]) +@router.get("/files", response_model=list[StoredFile]) async def list_files( - category: Optional[FileCategory] = None, - path: Optional[str] = None, + category: FileCategory | None = None, + path: str | None = None, recursive: bool = False, plugin_loader=Depends(get_plugin_loader), _ = Depends(has_permission("plugin:storage", "READ")) @@ -54,7 +55,7 @@ async def list_files( @router.post("/upload", response_model=StoredFile, status_code=201) async def upload_file( category: FileCategory = Form(...), - path: Optional[str] = Form(None), + path: str | None = Form(None), file: UploadFile = File(...), plugin_loader=Depends(get_plugin_loader), _ = Depends(has_permission("plugin:storage", "WRITE")) diff --git a/backend/src/api/routes/tasks.py b/backend/src/api/routes/tasks.py index 0bd21a01..53cf9b8b 100755 --- a/backend/src/api/routes/tasks.py +++ b/backend/src/api/routes/tasks.py @@ -5,20 +5,21 @@ # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [LLMProviderService] -from typing import List, Dict, Any, Optional -from fastapi import APIRouter, Depends, HTTPException, status, Query -from pydantic import BaseModel -from ...core.logger import belief_scope +from typing import Any -from ...core.task_manager import TaskManager, Task, TaskStatus, LogEntry +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic import BaseModel + +from ...core.config_manager import ConfigManager +from ...core.logger import belief_scope +from ...core.task_manager import Task, TaskManager, TaskStatus from ...core.task_manager.models import LogFilter, LogStats from ...dependencies import ( + get_config_manager, + get_current_user, get_task_manager, has_permission, - get_current_user, - get_config_manager, ) -from ...core.config_manager import ConfigManager from ...services.llm_prompt_templates import ( is_multimodal_model, normalize_llm_settings, @@ -36,15 +37,15 @@ TASK_TYPE_PLUGIN_MAP = { class CreateTaskRequest(BaseModel): plugin_id: str - params: Dict[str, Any] + params: dict[str, Any] class ResolveTaskRequest(BaseModel): - resolution_params: Dict[str, Any] + resolution_params: dict[str, Any] class ResumeTaskRequest(BaseModel): - passwords: Dict[str, str] + passwords: dict[str, str] # #region create_task [C:3] [TYPE Function] @@ -132,15 +133,15 @@ async def create_task( # @POST: Returns a list of tasks. # @RELATION CALLS -> [TaskManager] # @RELATION BINDS_TO -> [TASK_TYPE_PLUGIN_MAP] -@router.get("", response_model=List[Task]) +@router.get("", response_model=list[Task]) async def list_tasks( limit: int = 10, offset: int = 0, - status_filter: Optional[TaskStatus] = Query(None, alias="status"), - task_type: Optional[str] = Query( + status_filter: TaskStatus | None = Query(None, alias="status"), + task_type: str | None = Query( None, description="Task category: llm_validation, backup, migration" ), - plugin_id: Optional[List[str]] = Query( + plugin_id: list[str] | None = Query( None, description="Filter by plugin_id (repeatable query param)" ), completed_only: bool = Query( @@ -210,11 +211,11 @@ async def get_task( @router.get("/{task_id}/logs") async def get_task_logs( task_id: str, - level: Optional[str] = Query( + level: str | None = Query( None, description="Filter by log level (DEBUG, INFO, WARNING, ERROR)" ), - source: Optional[str] = Query(None, description="Filter by source component"), - search: Optional[str] = Query(None, description="Text search in message"), + source: str | None = Query(None, description="Filter by source component"), + search: str | None = Query(None, description="Text search in message"), offset: int = Query(0, ge=0, description="Number of logs to skip"), limit: int = Query( 100, ge=1, le=1000, description="Maximum number of logs to return" @@ -291,7 +292,7 @@ async def get_task_log_stats( # @PRE: task_id must exist. # @POST: Returns list of unique source names or raises 404. # @RELATION CALLS -> [TaskManager] -@router.get("/{task_id}/logs/sources", response_model=List[str]) +@router.get("/{task_id}/logs/sources", response_model=list[str]) async def get_task_log_sources( task_id: str, task_manager: TaskManager = Depends(get_task_manager), @@ -361,7 +362,7 @@ async def resume_task( # @RELATION CALLS -> [TaskManager] @router.delete("", status_code=status.HTTP_204_NO_CONTENT) async def clear_tasks( - status: Optional[TaskStatus] = None, + status: TaskStatus | None = None, task_manager: TaskManager = Depends(get_task_manager), _=Depends(has_permission("tasks", "WRITE")), ): diff --git a/backend/src/api/routes/translate/__init__.py b/backend/src/api/routes/translate/__init__.py index 726129cf..bd34d4a0 100644 --- a/backend/src/api/routes/translate/__init__.py +++ b/backend/src/api/routes/translate/__init__.py @@ -18,15 +18,17 @@ Re-exports the shared router and all route handler modules. All sub-modules register their handlers on the router at import time. """ +from . import ( + _correction_routes, # noqa: F401 — registers correction handlers + _dictionary_routes, # noqa: F401 — registers dictionary handlers + _job_routes, # noqa: F401 — registers job handlers + _metrics_routes, # noqa: F401 — registers metrics handlers + _preview_routes, # noqa: F401 — registers preview handlers + _run_list_routes, # noqa: F401 — registers run list/detail/csv handlers + _run_routes, # noqa: F401 — registers run handlers + _schedule_routes, # noqa: F401 — registers schedule handlers +) from ._router import router -from . import _job_routes # noqa: F401 — registers job handlers -from . import _run_routes # noqa: F401 — registers run handlers -from . import _run_list_routes # noqa: F401 — registers run list/detail/csv handlers -from . import _preview_routes # noqa: F401 — registers preview handlers -from . import _dictionary_routes # noqa: F401 — registers dictionary handlers -from . import _schedule_routes # noqa: F401 — registers schedule handlers -from . import _metrics_routes # noqa: F401 — registers metrics handlers -from . import _correction_routes # noqa: F401 — registers correction handlers __all__ = [ "router", diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py index 1493963c..b172b0ec 100644 --- a/backend/src/api/routes/translate/_correction_routes.py +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -2,24 +2,20 @@ # @BRIEF Term correction submission endpoints. # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional +from fastapi import Depends, HTTPException, status from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User +from ....core.logger import logger from ....dependencies import get_current_user, has_permission from ....plugins.translate.dictionary import DictionaryManager +from ....schemas.auth import User from ....schemas.translate import ( - TermCorrectionSubmit, TermCorrectionBulkSubmit, - CorrectionSubmitResponse, + TermCorrectionSubmit, ) - from ._router import router - # ============================================================ # Corrections # ============================================================ diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py index 84fe01a7..e92650e1 100644 --- a/backend/src/api/routes/translate/_dictionary_routes.py +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -2,26 +2,21 @@ # @BRIEF Terminology Dictionary CRUD, entries management, and import routes. # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional +from fastapi import Depends, HTTPException, Query, status from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User +from ....core.logger import belief_scope, logger from ....dependencies import get_current_user, has_permission from ....plugins.translate.dictionary import DictionaryManager +from ....schemas.auth import User from ....schemas.translate import ( DictionaryCreate, DictionaryEntryCreate, - DictionaryEntryResponse, DictionaryImport, - DictionaryResponse, ) - -from ._router import router from ._helpers import _dict_to_response, _get_dictionary_entry_counts - +from ._router import router # ============================================================ # Terminology Dictionaries diff --git a/backend/src/api/routes/translate/_helpers.py b/backend/src/api/routes/translate/_helpers.py index 25102312..2ee66965 100644 --- a/backend/src/api/routes/translate/_helpers.py +++ b/backend/src/api/routes/translate/_helpers.py @@ -3,7 +3,8 @@ # @LAYER: API # @RELATION DEPENDS_ON -> [models.translate] -from typing import Any, Dict, List +from typing import Any + from sqlalchemy.orm import Session @@ -55,8 +56,9 @@ def _dict_to_response(d: Any, entry_count: int = 0) -> dict: # #region _get_dictionary_entry_counts [C:2] [TYPE Function] # @BRIEF Get entry counts for a list of dictionary IDs. -def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]: +def _get_dictionary_entry_counts(db: Session, dict_ids: list[str]) -> dict[str, int]: from sqlalchemy import func + from ....models.translate import DictionaryEntry counts = ( db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id)) diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py index 6a84ce15..aeda6fda 100644 --- a/backend/src/api/routes/translate/_job_routes.py +++ b/backend/src/api/routes/translate/_job_routes.py @@ -2,31 +2,31 @@ # @BRIEF Translation Job CRUD and datasource column routes. # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional + +from fastapi import Depends, HTTPException, Query, status from sqlalchemy.orm import Session -from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User -from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager +from ....core.database import get_db +from ....core.logger import belief_scope, logger +from ....dependencies import get_config_manager, get_current_user, has_permission from ....plugins.translate.service import ( TranslateJobService, job_to_response, +) +from ....plugins.translate.service import ( get_datasource_columns as fetch_datasource_columns_service, ) +from ....schemas.auth import User from ....schemas.translate import ( - TranslateJobCreate, - TranslateJobUpdate, - TranslateJobResponse, DatasourceColumnsResponse, DuplicateJobResponse, + TranslateJobCreate, + TranslateJobResponse, + TranslateJobUpdate, ) - from ._router import router - # ============================================================ # Translation Job CRUD # ============================================================ @@ -35,11 +35,11 @@ from ._router import router # @BRIEF List all translation jobs. # @PRE: User has translate.job.view permission. # @POST: Returns list of translation jobs. -@router.get("/jobs", response_model=List[TranslateJobResponse]) +@router.get("/jobs", response_model=list[TranslateJobResponse]) async def list_jobs( page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100), - status: Optional[str] = Query(None), + status: str | None = Query(None), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.job", "VIEW")), db: Session = Depends(get_db), @@ -199,7 +199,7 @@ async def duplicate_job( @router.get("/datasources") async def list_datasources( env_id: str = Query(..., description="Superset environment ID"), - search: Optional[str] = Query(None, description="Search by table name"), + search: str | None = Query(None, description="Search by table name"), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.job", "VIEW")), db: Session = Depends(get_db), @@ -212,7 +212,7 @@ async def list_datasources( datasets = service.fetch_available_datasources(env_id, search) return datasets except Exception as e: - logger.explore(f"list_datasources failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("list_datasources failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) @router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse) @@ -238,7 +238,7 @@ async def get_datasource_columns( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.explore(f"get_datasource_columns failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("get_datasource_columns failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Failed to fetch datasource columns from Superset: {e}", diff --git a/backend/src/api/routes/translate/_metrics_routes.py b/backend/src/api/routes/translate/_metrics_routes.py index 6011d4ea..167997a2 100644 --- a/backend/src/api/routes/translate/_metrics_routes.py +++ b/backend/src/api/routes/translate/_metrics_routes.py @@ -2,19 +2,17 @@ # @BRIEF Translation Metrics endpoints. # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional + +from fastapi import Depends, HTTPException, Query, status from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User +from ....core.logger import logger from ....dependencies import get_current_user, has_permission from ....plugins.translate.metrics import TranslationMetrics - +from ....schemas.auth import User from ._router import router - # ============================================================ # Metrics # ============================================================ @@ -25,7 +23,7 @@ from ._router import router # @POST: Returns metrics data. @router.get("/metrics") async def get_metrics( - job_id: Optional[str] = Query(None), + job_id: str | None = Query(None), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.metrics", "VIEW")), db: Session = Depends(get_db), diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py index 61ae9e3f..30206b23 100644 --- a/backend/src/api/routes/translate/_preview_routes.py +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -2,26 +2,23 @@ # @BRIEF Translation Preview session management routes. # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional + +from fastapi import Depends, HTTPException, status from sqlalchemy.orm import Session -from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User -from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager +from ....core.database import get_db +from ....core.logger import logger +from ....dependencies import get_config_manager, get_current_user, has_permission from ....plugins.translate.preview import TranslationPreview +from ....schemas.auth import User from ....schemas.translate import ( PreviewRequest, - PreviewRowUpdate, - PreviewAcceptResponse, PreviewRow, + PreviewRowUpdate, ) - from ._router import router - # ============================================================ # Preview # ============================================================ @@ -56,7 +53,7 @@ async def preview_translation( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.explore(f"preview_translation failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("preview_translation failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}") # #endregion preview_translation @@ -151,7 +148,7 @@ async def apply_preview( # @BRIEF Get records for a preview session. # @PRE: User has translate.job.view permission. # @POST: Returns list of preview records. -@router.get("/preview/{session_id}/records", response_model=List[PreviewRow]) +@router.get("/preview/{session_id}/records", response_model=list[PreviewRow]) async def get_preview_records( session_id: str, current_user: User = Depends(get_current_user), @@ -161,7 +158,7 @@ async def get_preview_records( """Get records for a preview session.""" logger.reason(f"get_preview_records — Session: {session_id}, User: {current_user.username}", extra={"src": "translate_routes"}) try: - from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord + from ....models.translate import TranslationPreviewRecord, TranslationPreviewSession session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() if not session: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") @@ -186,7 +183,7 @@ async def get_preview_records( except HTTPException: raise except Exception as e: - logger.explore(f"get_preview_records failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("get_preview_records failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_preview_records diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index 7d1daf40..5677ab0d 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -2,21 +2,19 @@ # @BRIEF Translation Run listing, detail, and CSV download routes (cross-job). # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional + +from fastapi import Depends, HTTPException, Query, status from sqlalchemy.orm import Session -from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User -from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager -from ....plugins.translate.orchestrator import TranslationOrchestrator +from ....core.database import get_db +from ....core.logger import logger +from ....dependencies import get_config_manager, get_current_user, has_permission from ....plugins.translate.events import TranslationEventLog - +from ....plugins.translate.orchestrator import TranslationOrchestrator +from ....schemas.auth import User from ._router import router - # ============================================================ # List Runs (cross-job) # ============================================================ @@ -27,12 +25,12 @@ from ._router import router # @POST: Returns paginated list of runs. @router.get("/runs") async def list_runs( - job_id: Optional[str] = Query(None), - run_status: Optional[str] = Query(None), - trigger_type: Optional[str] = Query(None), - created_by: Optional[str] = Query(None), - date_from: Optional[str] = Query(None), - date_to: Optional[str] = Query(None), + job_id: str | None = Query(None), + run_status: str | None = Query(None), + trigger_type: str | None = Query(None), + created_by: str | None = Query(None), + date_from: str | None = Query(None), + date_to: str | None = Query(None), page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100), current_user: User = Depends(get_current_user), @@ -100,7 +98,7 @@ async def list_runs( return {"items": items, "total": total, "page": page, "page_size": page_size} except Exception as e: - logger.explore(f"list_runs failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("list_runs failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion list_runs @@ -166,11 +164,13 @@ async def download_skipped_csv( """Download skipped translation records as CSV.""" logger.reason(f"download_skipped_csv — Run: {run_id}, User: {current_user.username}", extra={"src": "translate_routes"}) try: - from ....models.translate import TranslationRecord - from fastapi.responses import StreamingResponse import csv import io + from fastapi.responses import StreamingResponse + + from ....models.translate import TranslationRecord + records = ( db.query(TranslationRecord) .filter( @@ -197,7 +197,7 @@ async def download_skipped_csv( headers={"Content-Disposition": f"attachment; filename=skipped_{run_id}.csv"}, ) except Exception as e: - logger.explore(f"download_skipped_csv failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("download_skipped_csv failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion download_skipped_csv diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index 93d97f93..56d05973 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -2,22 +2,19 @@ # @BRIEF Translation Run execution, history, status, records and batches routes. # @LAYER: API -from datetime import datetime, timezone -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional +from datetime import UTC, datetime + +from fastapi import Depends, HTTPException, Query, status from sqlalchemy.orm import Session -from ....core.database import get_db, SessionLocal -from ....core.logger import logger, belief_scope -from ....schemas.auth import User -from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager +from ....core.database import SessionLocal, get_db +from ....core.logger import logger +from ....dependencies import get_config_manager, get_current_user, has_permission from ....plugins.translate.orchestrator import TranslationOrchestrator -from ....plugins.translate.events import TranslationEventLog - -from ._router import router +from ....schemas.auth import User from ._helpers import _run_to_response - +from ._router import router # ============================================================ # Translation Run / Execute @@ -119,7 +116,7 @@ async def run_translation( if fb_run: fb_run.status = "FAILED" fb_run.error_message = f"Background execute error: {bg_err}" - fb_run.completed_at = datetime.now(timezone.utc) + fb_run.completed_at = datetime.now(UTC) fb_db.commit() logger.reason( "Background execute: run marked FAILED", @@ -147,7 +144,7 @@ async def run_translation( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.explore(f"run_translation failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("run_translation failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}") # #endregion run_translation @@ -173,7 +170,7 @@ async def retry_run( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.explore(f"retry_run failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("retry_run failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}") # #endregion retry_run @@ -199,7 +196,7 @@ async def retry_insert( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - logger.explore(f"retry_insert failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("retry_insert failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}") # #endregion retry_insert @@ -283,7 +280,7 @@ async def get_run_records( run_id: str, page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=500), - status: Optional[str] = Query(None), + status: str | None = Query(None), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.history", "VIEW")), db: Session = Depends(get_db), @@ -340,7 +337,7 @@ async def get_batches( for b in batches ] except Exception as e: - logger.explore(f"get_batches failed", extra={"src": "translate_routes", "error": str(e)}) + logger.explore("get_batches failed", extra={"src": "translate_routes", "error": str(e)}) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_batches diff --git a/backend/src/api/routes/translate/_schedule_routes.py b/backend/src/api/routes/translate/_schedule_routes.py index 7204d1ad..b70a205b 100644 --- a/backend/src/api/routes/translate/_schedule_routes.py +++ b/backend/src/api/routes/translate/_schedule_routes.py @@ -2,21 +2,18 @@ # @BRIEF Translation Schedule management routes. # @LAYER: API -from fastapi import APIRouter, Depends, HTTPException, status, Query -from typing import Any, Dict, List, Optional +from fastapi import Depends, HTTPException, Query, status from sqlalchemy.orm import Session -from ....core.database import get_db -from ....core.logger import logger, belief_scope -from ....schemas.auth import User -from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager +from ....core.database import get_db +from ....core.logger import logger +from ....dependencies import get_config_manager, get_current_user, has_permission from ....plugins.translate.scheduler import TranslationScheduler -from ....schemas.translate import ScheduleConfig, ScheduleResponse - +from ....schemas.auth import User +from ....schemas.translate import ScheduleConfig from ._router import router - # ============================================================ # Schedule # ============================================================ diff --git a/backend/src/app.py b/backend/src/app.py index 44c4baff..c932b795 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -11,46 +11,50 @@ # @DATA_CONTRACT: [HTTP Request | WS Message] -> [HTTP Response | JSON Log Stream] import os from pathlib import Path + # project_root is used for static files mounting project_root = Path(__file__).resolve().parent.parent.parent -from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPException -from starlette.middleware.sessions import SessionMiddleware -from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse import asyncio -from .dependencies import get_task_manager, get_scheduler_service -from .core.encryption_key import ensure_encryption_key -from .core.utils.network import NetworkError -from .core.logger import logger, belief_scope -from .core.cot_logger import seed_trace_id -from .core.database import AuthSessionLocal -from .core.auth.security import get_password_hash -from .models.auth import User, Role + +from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles +from starlette.middleware.sessions import SessionMiddleware + +from .api import auth from .api.routes import ( - plugins, - tasks, - settings, - environments, - mappings, - migration, - connections, - git, - storage, admin, - llm, - dashboards, - datasets, - reports, assistant, clean_release, clean_release_v2, - profile, - health, + connections, + dashboards, dataset_review, + datasets, + environments, + git, + health, + llm, + mappings, + migration, + plugins, + profile, + reports, + settings, + storage, + tasks, translate, ) -from .api import auth +from .core.auth.security import get_password_hash +from .core.cot_logger import seed_trace_id +from .core.database import AuthSessionLocal +from .core.encryption_key import ensure_encryption_key +from .core.logger import belief_scope, logger +from .core.utils.network import NetworkError +from .dependencies import get_scheduler_service, get_task_manager +from .models.auth import Role, User + # #region FastAPI_App [C:3] [TYPE Global] [SEMANTICS app, fastapi, instance, route-registry] # @BRIEF Canonical FastAPI application instance for route, middleware, and websocket registration. # @RELATION DEPENDS_ON -> [ApiRoutesModule] @@ -141,6 +145,7 @@ async def shutdown_event(): # @BRIEF Configure application-wide middleware (Session, CORS). # Configure Session Middleware (required by Authlib for OAuth2 flow) from .core.auth.config import auth_config + app.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY) # Configure CORS app.add_middleware( @@ -193,6 +198,7 @@ async def log_requests(request: Request, call_next): # Seed trace_id on every request — MUST be the outermost middleware so trace_id # is seeded before log_requests (or any other middleware) runs. from .core.middleware.trace import TraceContextMiddleware + app.add_middleware(TraceContextMiddleware) # #region API_Routes [C:3] [TYPE Block] # @BRIEF Register all FastAPI route groups exposed by the application entrypoint. @@ -425,4 +431,4 @@ else: } # #endregion read_root # #endregion StaticFiles -# #endregion AppModule \ No newline at end of file +# #endregion AppModule diff --git a/backend/src/core/__tests__/test_config_manager_compat.py b/backend/src/core/__tests__/test_config_manager_compat.py index 4cbf1a57..fd3a9dc2 100644 --- a/backend/src/core/__tests__/test_config_manager_compat.py +++ b/backend/src/core/__tests__/test_config_manager_compat.py @@ -3,8 +3,11 @@ # @LAYER: Domain # @RELATION VERIFIES -> ConfigManager from types import SimpleNamespace + from src.core.config_manager import ConfigManager from src.core.config_models import AppConfig, Environment, GlobalSettings + + # #region test_get_payload_preserves_legacy_sections [TYPE Function] # @RELATION BINDS_TO -> TestConfigManagerCompat # @BRIEF Ensure get_payload merges typed config into raw payload without dropping legacy sections. @@ -207,4 +210,4 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa assert committed["value"] is True assert closed["value"] is True # #endregion test_load_config_syncs_environment_records_from_existing_db_payload -# #endregion TestConfigManagerCompat \ No newline at end of file +# #endregion TestConfigManagerCompat diff --git a/backend/src/core/__tests__/test_native_filters.py b/backend/src/core/__tests__/test_native_filters.py index 546a6459..ec8de610 100644 --- a/backend/src/core/__tests__/test_native_filters.py +++ b/backend/src/core/__tests__/test_native_filters.py @@ -6,21 +6,22 @@ # @RELATION [BINDS_TO] ->[FilterState, ParsedNativeFilters, ExtraFormDataMerge] import json from unittest.mock import MagicMock -import pytest -from src.core.superset_client import SupersetClient -from src.core.async_superset_client import AsyncSupersetClient + from src.core.config_models import Environment +from src.core.superset_client import SupersetClient from src.core.utils.superset_context_extractor import ( SupersetContextExtractor, SupersetParsedContext, sanitize_imported_filter_for_assistant, ) from src.models.filter_state import ( + ExtraFormDataMerge, FilterState, NativeFilterDataMask, ParsedNativeFilters, - ExtraFormDataMerge, ) + + # #region _make_environment [TYPE Function] # @RELATION BINDS_TO -> NativeFilterExtractionTests def _make_environment() -> Environment: @@ -568,4 +569,4 @@ def test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values(): assert result["raw_value_masked"] is True assert result["normalized_value"] == {"filter_clauses": []} # #endregion test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values -# #endregion NativeFilterExtractionTests \ No newline at end of file +# #endregion NativeFilterExtractionTests diff --git a/backend/src/core/__tests__/test_superset_preview_pipeline.py b/backend/src/core/__tests__/test_superset_preview_pipeline.py index f9d7fe4b..cb2ce0f4 100644 --- a/backend/src/core/__tests__/test_superset_preview_pipeline.py +++ b/backend/src/core/__tests__/test_superset_preview_pipeline.py @@ -4,13 +4,17 @@ # @RELATION [BINDS_TO] ->[AsyncNetworkModule] import json from unittest.mock import MagicMock + import httpx import pytest import requests + from src.core.config_models import Environment from src.core.superset_client import SupersetClient from src.core.utils.async_network import AsyncAPIClient from src.core.utils.network import APIClient, DashboardNotFoundError, SupersetAPIError + + # #region _make_environment [TYPE Function] # @RELATION BINDS_TO -> SupersetPreviewPipelineTests def _make_environment() -> Environment: @@ -454,4 +458,4 @@ async def test_async_network_404_mapping_translates_dashboard_endpoints(): finally: await client.aclose() # #endregion test_async_network_404_mapping_translates_dashboard_endpoints -# #endregion SupersetPreviewPipelineTests \ No newline at end of file +# #endregion SupersetPreviewPipelineTests diff --git a/backend/src/core/__tests__/test_superset_profile_lookup.py b/backend/src/core/__tests__/test_superset_profile_lookup.py index 135784b0..41bb2789 100644 --- a/backend/src/core/__tests__/test_superset_profile_lookup.py +++ b/backend/src/core/__tests__/test_superset_profile_lookup.py @@ -6,13 +6,17 @@ import json import sys from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any + import pytest + backend_dir = str(Path(__file__).parent.parent.parent.parent.resolve()) if backend_dir not in sys.path: sys.path.insert(0, backend_dir) from src.core.superset_profile_lookup import SupersetAccountLookupAdapter from src.core.utils.network import AuthenticationError, SupersetAPIError + + # [/SECTION] # #region _RecordingNetworkClient [TYPE Class] [C:2] # @RELATION BINDS_TO -> TestSupersetProfileLookup @@ -23,9 +27,9 @@ class _RecordingNetworkClient: # @PURPOSE: Initializes scripted network responses. # @PRE: scripted_responses is ordered per expected request sequence. # @POST: Instance stores response script and captures subsequent request calls. - def __init__(self, scripted_responses: List[Any]): + def __init__(self, scripted_responses: list[Any]): self._scripted_responses = scripted_responses - self.calls: List[Dict[str, Any]] = [] + self.calls: list[dict[str, Any]] = [] # #endregion __init__ # #region request [TYPE Function] # @PURPOSE: Mimics APIClient.request while capturing call arguments. @@ -35,9 +39,9 @@ class _RecordingNetworkClient: self, method: str, endpoint: str, - params: Optional[Dict[str, Any]] = None, + params: dict[str, Any] | None = None, **kwargs, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: self.calls.append( { "method": method, @@ -116,4 +120,4 @@ def test_get_users_page_uses_fallback_endpoint_when_primary_fails(): "/security/users", ] # #endregion test_get_users_page_uses_fallback_endpoint_when_primary_fails -# #endregion TestSupersetProfileLookup \ No newline at end of file +# #endregion TestSupersetProfileLookup diff --git a/backend/src/core/__tests__/test_throttled_scheduler.py b/backend/src/core/__tests__/test_throttled_scheduler.py index 90cd9d96..57e56b55 100644 --- a/backend/src/core/__tests__/test_throttled_scheduler.py +++ b/backend/src/core/__tests__/test_throttled_scheduler.py @@ -1,6 +1,8 @@ -import pytest -from datetime import time, date, datetime, timedelta +from datetime import date, datetime, time + from src.core.scheduler import ThrottledSchedulerConfigurator + + # #region test_throttled_scheduler [TYPE Module] [C:3] [SEMANTICS test, scheduler, throttle, unit] # @RELATION BELONGS_TO -> SrcRoot # @BRIEF Unit tests for ThrottledSchedulerConfigurator distribution logic. @@ -112,4 +114,4 @@ def test_calculate_schedule_very_small_window(): assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s assert schedule[2] == datetime(2024, 1, 1, 1, 0, 1) # #endregion test_calculate_schedule_very_small_window -# #endregion test_throttled_scheduler \ No newline at end of file +# #endregion test_throttled_scheduler diff --git a/backend/src/core/async_superset_client.py b/backend/src/core/async_superset_client.py index 62c065e7..49b84942 100644 --- a/backend/src/core/async_superset_client.py +++ b/backend/src/core/async_superset_client.py @@ -4,11 +4,15 @@ import asyncio import json import re -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, cast + from .config_models import Environment -from .logger import logger as app_logger, belief_scope +from .logger import belief_scope +from .logger import logger as app_logger from .superset_client import SupersetClient from .utils.async_network import AsyncAPIClient + + # #region AsyncSupersetClient [C:3] [TYPE Class] # @BRIEF Async sibling of SupersetClient for dashboard read paths. # @RELATION INHERITS -> [SupersetClient] @@ -50,8 +54,8 @@ class AsyncSupersetClient(SupersetClient): # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] # @RELATION: [CALLS] -> [AsyncAPIClient.request] async def get_dashboards_page_async( - self, query: Optional[Dict] = None - ) -> Tuple[int, List[Dict]]: + self, query: dict | None = None + ) -> tuple[int, list[dict]]: with belief_scope("AsyncSupersetClient.get_dashboards_page_async"): validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: @@ -68,7 +72,7 @@ class AsyncSupersetClient(SupersetClient): "owners", ] response_json = cast( - Dict[str, Any], + dict[str, Any], await self.network.request( method="GET", endpoint="/dashboard/", @@ -84,26 +88,26 @@ class AsyncSupersetClient(SupersetClient): # @POST: Returns raw dashboard payload from Superset API. # @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict] # @RELATION: [CALLS] ->[AsyncAPIClient.request] - async def get_dashboard_async(self, dashboard_id: int) -> Dict: + async def get_dashboard_async(self, dashboard_id: int) -> dict: with belief_scope( "AsyncSupersetClient.get_dashboard_async", f"id={dashboard_id}" ): response = await self.network.request( method="GET", endpoint=f"/dashboard/{dashboard_id}" ) - return cast(Dict, response) + return cast(dict, response) # #endregion get_dashboard_async # #region get_chart_async [TYPE Function] [C:3] # @PURPOSE: Fetch one chart payload asynchronously. # @POST: Returns raw chart payload from Superset API. # @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict] # @RELATION: [CALLS] ->[AsyncAPIClient.request] - async def get_chart_async(self, chart_id: int) -> Dict: + async def get_chart_async(self, chart_id: int) -> dict: with belief_scope("AsyncSupersetClient.get_chart_async", f"id={chart_id}"): response = await self.network.request( method="GET", endpoint=f"/chart/{chart_id}" ) - return cast(Dict, response) + return cast(dict, response) # #endregion get_chart_async # #region get_dashboard_detail_async [TYPE Function] [C:3] # @PURPOSE: Fetch dashboard detail asynchronously with concurrent charts/datasets requests. @@ -111,17 +115,17 @@ class AsyncSupersetClient(SupersetClient): # @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict] # @RELATION: [CALLS] ->[get_dashboard_async] # @RELATION: [CALLS] ->[get_chart_async] - async def get_dashboard_detail_async(self, dashboard_id: int) -> Dict: + async def get_dashboard_detail_async(self, dashboard_id: int) -> dict: with belief_scope( "AsyncSupersetClient.get_dashboard_detail_async", f"id={dashboard_id}" ): dashboard_response = await self.get_dashboard_async(dashboard_id) dashboard_data = dashboard_response.get("result", dashboard_response) - charts: List[Dict] = [] - datasets: List[Dict] = [] + charts: list[dict] = [] + datasets: list[dict] = [] def extract_dataset_id_from_form_data( - form_data: Optional[Dict], - ) -> Optional[int]: + form_data: dict | None, + ) -> int | None: if not isinstance(form_data, dict): return None datasource = form_data.get("datasource") @@ -394,7 +398,7 @@ class AsyncSupersetClient(SupersetClient): # @PURPOSE: Fetch stored dashboard permalink state asynchronously. # @POST: Returns dashboard permalink state payload from Superset API. # @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict] - async def get_dashboard_permalink_state_async(self, permalink_key: str) -> Dict: + async def get_dashboard_permalink_state_async(self, permalink_key: str) -> dict: with belief_scope( "AsyncSupersetClient.get_dashboard_permalink_state_async", f"key={permalink_key}", @@ -402,7 +406,7 @@ class AsyncSupersetClient(SupersetClient): response = await self.network.request( method="GET", endpoint=f"/dashboard/permalink/{permalink_key}" ) - return cast(Dict, response) + return cast(dict, response) # #endregion get_dashboard_permalink_state_async # #region get_native_filter_state_async [TYPE Function] [C:2] # @PURPOSE: Fetch stored native filter state asynchronously. @@ -410,7 +414,7 @@ class AsyncSupersetClient(SupersetClient): # @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] async def get_native_filter_state_async( self, dashboard_id: int, filter_state_key: str - ) -> Dict: + ) -> dict: with belief_scope( "AsyncSupersetClient.get_native_filter_state_async", f"dashboard={dashboard_id}, key={filter_state_key}", @@ -419,7 +423,7 @@ class AsyncSupersetClient(SupersetClient): method="GET", endpoint=f"/dashboard/{dashboard_id}/filter_state/{filter_state_key}", ) - return cast(Dict, response) + return cast(dict, response) # #endregion get_native_filter_state_async # #region extract_native_filters_from_permalink_async [TYPE Function] [C:3] # @PURPOSE: Extract native filters dataMask from a permalink key asynchronously. @@ -428,7 +432,7 @@ class AsyncSupersetClient(SupersetClient): # @RELATION: [CALLS] ->[get_dashboard_permalink_state_async] async def extract_native_filters_from_permalink_async( self, permalink_key: str - ) -> Dict: + ) -> dict: with belief_scope( "AsyncSupersetClient.extract_native_filters_from_permalink_async", f"key={permalink_key}", @@ -463,7 +467,7 @@ class AsyncSupersetClient(SupersetClient): # @RELATION: [CALLS] ->[get_native_filter_state_async] async def extract_native_filters_from_key_async( self, dashboard_id: int, filter_state_key: str - ) -> Dict: + ) -> dict: with belief_scope( "AsyncSupersetClient.extract_native_filters_from_key_async", f"dashboard={dashboard_id}, key={filter_state_key}", @@ -515,7 +519,7 @@ class AsyncSupersetClient(SupersetClient): # @DATA_CONTRACT: Input[url: str] -> Output[Dict] # @RELATION: [CALLS] ->[extract_native_filters_from_permalink_async] # @RELATION: [CALLS] ->[extract_native_filters_from_key_async] - async def parse_dashboard_url_for_filters_async(self, url: str) -> Dict: + async def parse_dashboard_url_for_filters_async(self, url: str) -> dict: with belief_scope( "AsyncSupersetClient.parse_dashboard_url_for_filters_async", f"url={url}" ): @@ -609,4 +613,4 @@ class AsyncSupersetClient(SupersetClient): return result # #endregion parse_dashboard_url_for_filters_async # #endregion AsyncSupersetClient -# #endregion AsyncSupersetClientModule \ No newline at end of file +# #endregion AsyncSupersetClientModule diff --git a/backend/src/core/auth/__tests__/test_auth.py b/backend/src/core/auth/__tests__/test_auth.py index acd60d48..837462fc 100644 --- a/backend/src/core/auth/__tests__/test_auth.py +++ b/backend/src/core/auth/__tests__/test_auth.py @@ -4,18 +4,21 @@ # @RELATION VERIFIES -> AuthPackage import sys from pathlib import Path + # Add src to path sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src")) import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from src.core.database import Base -# Import all models to ensure they are registered with Base before create_all - must import both auth and mapping to ensure Base knows about all tables -from src.models import mapping, auth, task, report -from src.models.auth import User, Role, Permission, ADGroupMapping -from src.services.auth_service import AuthService + from src.core.auth.repository import AuthRepository -from src.core.auth.security import verify_password, get_password_hash +from src.core.auth.security import get_password_hash, verify_password +from src.core.database import Base + +# Import all models to ensure they are registered with Base before create_all - must import both auth and mapping to ensure Base knows about all tables +from src.models.auth import ADGroupMapping, Permission, Role, User +from src.services.auth_service import AuthService + # Create in-memory SQLite database for testing SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" engine = create_engine( @@ -252,4 +255,4 @@ def test_provision_adfs_user_existing(auth_service, auth_repo): assert user.username == "existingadfs@domain.com" assert len(user.roles) == 0 # No matching group mappings # #endregion test_auth -# #endregion test_provision_adfs_user_existing \ No newline at end of file +# #endregion test_provision_adfs_user_existing diff --git a/backend/src/core/auth/config.py b/backend/src/core/auth/config.py index a83eb53f..3737638f 100644 --- a/backend/src/core/auth/config.py +++ b/backend/src/core/auth/config.py @@ -9,6 +9,7 @@ from pydantic import Field from pydantic_settings import BaseSettings + # #region AuthConfig [TYPE Class] # @BRIEF Holds authentication-related settings. # @PRE: Environment variables may be provided via .env file. @@ -31,7 +32,7 @@ class AuthConfig(BaseSettings): ADFS_CLIENT_ID: str = Field(default="", env="ADFS_CLIENT_ID") ADFS_CLIENT_SECRET: str = Field(default="", env="ADFS_CLIENT_SECRET") ADFS_METADATA_URL: str = Field(default="", env="ADFS_METADATA_URL") - + class Config: env_file = ".env" extra = "ignore" diff --git a/backend/src/core/auth/jwt.py b/backend/src/core/auth/jwt.py index 28211d8b..147d3dcb 100644 --- a/backend/src/core/auth/jwt.py +++ b/backend/src/core/auth/jwt.py @@ -8,10 +8,11 @@ # @INVARIANT: Tokens must include expiration time and user identifier. from datetime import datetime, timedelta -from typing import Optional + from jose import jwt -from .config import auth_config + from ..logger import belief_scope +from .config import auth_config # #region create_access_token [TYPE Function] @@ -20,7 +21,7 @@ from ..logger import belief_scope # @POST: Returns a signed JWT string. # @RELATION DEPENDS_ON -> [auth_config] # -def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str: +def create_access_token(data: dict, expires_delta: timedelta | None = None) -> str: with belief_scope("create_access_token"): to_encode = data.copy() if expires_delta: diff --git a/backend/src/core/auth/logger.py b/backend/src/core/auth/logger.py index 3f4c7155..a4de434d 100644 --- a/backend/src/core/auth/logger.py +++ b/backend/src/core/auth/logger.py @@ -6,9 +6,11 @@ # # @INVARIANT: Must not log sensitive data like passwords or full tokens. -from ..logger import logger, belief_scope from datetime import datetime +from ..logger import belief_scope, logger + + # #region log_security_event [TYPE Function] # @BRIEF Logs a security-related event for audit trails. # @PRE: event_type and username are strings. @@ -23,4 +25,4 @@ def log_security_event(event_type: str, username: str, details: dict = None): logger.info(msg) # #endregion log_security_event -# #endregion AuthLoggerModule \ No newline at end of file +# #endregion AuthLoggerModule diff --git a/backend/src/core/auth/oauth.py b/backend/src/core/auth/oauth.py index ac568e1b..d35499fe 100644 --- a/backend/src/core/auth/oauth.py +++ b/backend/src/core/auth/oauth.py @@ -8,6 +8,7 @@ # @INVARIANT: Must use secure OIDC flows. from authlib.integrations.starlette_client import OAuth + from .config import auth_config # #region oauth [TYPE Variable] @@ -48,4 +49,4 @@ def is_adfs_configured() -> bool: # Initial registration register_adfs() -# #endregion AuthOauthModule \ No newline at end of file +# #endregion AuthOauthModule diff --git a/backend/src/core/auth/repository.py b/backend/src/core/auth/repository.py index 93b5eb5a..a9f5e171 100644 --- a/backend/src/core/auth/repository.py +++ b/backend/src/core/auth/repository.py @@ -9,11 +9,14 @@ # @PRE: Database connection is active. # @POST: Provides valid access to identity data. # @SIDE_EFFECT: Executes database read queries through the injected SQLAlchemy session boundary. -from typing import List, Optional + from sqlalchemy.orm import Session, selectinload -from ...models.auth import Permission, Role, User, ADGroupMapping + +from ...models.auth import ADGroupMapping, Permission, Role, User from ...models.profile import UserDashboardPreference from ..logger import belief_scope, logger + + # #region AuthRepository [TYPE Class] # @BRIEF Provides low-level CRUD operations for identity and authorization records. # @PRE: Database session is bound. @@ -28,7 +31,7 @@ class AuthRepository: # @PRE: user_id is a valid UUID string. # @POST: Returns User object if found, else None. # @RELATION: DEPENDS_ON -> [User] - def get_user_by_id(self, user_id: str) -> Optional[User]: + def get_user_by_id(self, user_id: str) -> User | None: with belief_scope("AuthRepository.get_user_by_id"): logger.reason(f"Fetching user by id: {user_id}") result = self.db.query(User).filter(User.id == user_id).first() @@ -40,7 +43,7 @@ class AuthRepository: # @PRE: username is a non-empty string. # @POST: Returns User object if found, else None. # @RELATION: DEPENDS_ON -> [User] - def get_user_by_username(self, username: str) -> Optional[User]: + def get_user_by_username(self, username: str) -> User | None: with belief_scope("AuthRepository.get_user_by_username"): logger.reason(f"Fetching user by username: {username}") result = self.db.query(User).filter(User.username == username).first() @@ -51,7 +54,7 @@ class AuthRepository: # @PURPOSE: Retrieve role by UUID with permissions preloaded. # @RELATION: DEPENDS_ON -> [Role] # @RELATION: DEPENDS_ON -> [Permission] - def get_role_by_id(self, role_id: str) -> Optional[Role]: + def get_role_by_id(self, role_id: str) -> Role | None: with belief_scope("AuthRepository.get_role_by_id"): return ( self.db.query(Role) @@ -63,14 +66,14 @@ class AuthRepository: # #region get_role_by_name [TYPE Function] # @PURPOSE: Retrieve role by unique name. # @RELATION: DEPENDS_ON -> [Role] - def get_role_by_name(self, name: str) -> Optional[Role]: + def get_role_by_name(self, name: str) -> Role | None: with belief_scope("AuthRepository.get_role_by_name"): return self.db.query(Role).filter(Role.name == name).first() # #endregion get_role_by_name # #region get_permission_by_id [TYPE Function] # @PURPOSE: Retrieve permission by UUID. # @RELATION: DEPENDS_ON -> [Permission] - def get_permission_by_id(self, permission_id: str) -> Optional[Permission]: + def get_permission_by_id(self, permission_id: str) -> Permission | None: with belief_scope("AuthRepository.get_permission_by_id"): return ( self.db.query(Permission).filter(Permission.id == permission_id).first() @@ -81,7 +84,7 @@ class AuthRepository: # @RELATION: DEPENDS_ON -> [Permission] def get_permission_by_resource_action( self, resource: str, action: str - ) -> Optional[Permission]: + ) -> Permission | None: with belief_scope("AuthRepository.get_permission_by_resource_action"): return ( self.db.query(Permission) @@ -92,7 +95,7 @@ class AuthRepository: # #region list_permissions [TYPE Function] # @PURPOSE: List all system permissions. # @RELATION: DEPENDS_ON -> [Permission] - def list_permissions(self) -> List[Permission]: + def list_permissions(self) -> list[Permission]: with belief_scope("AuthRepository.list_permissions"): return self.db.query(Permission).all() # #endregion list_permissions @@ -101,7 +104,7 @@ class AuthRepository: # @RELATION: DEPENDS_ON -> [UserDashboardPreference] def get_user_dashboard_preference( self, user_id: str - ) -> Optional[UserDashboardPreference]: + ) -> UserDashboardPreference | None: with belief_scope("AuthRepository.get_user_dashboard_preference"): return ( self.db.query(UserDashboardPreference) @@ -115,7 +118,7 @@ class AuthRepository: # @POST: Returns a list of Role objects mapped to the provided AD groups. # @RELATION: DEPENDS_ON -> [Role] # @RELATION: DEPENDS_ON -> [ADGroupMapping] - def get_roles_by_ad_groups(self, groups: List[str]) -> List[Role]: + def get_roles_by_ad_groups(self, groups: list[str]) -> list[Role]: with belief_scope("AuthRepository.get_roles_by_ad_groups"): logger.reason(f"Fetching roles for AD groups: {groups}") if not groups: @@ -128,4 +131,4 @@ class AuthRepository: ) # #endregion get_roles_by_ad_groups # #endregion AuthRepository -# #endregion AuthRepositoryModule \ No newline at end of file +# #endregion AuthRepositoryModule diff --git a/backend/src/core/auth/security.py b/backend/src/core/auth/security.py index 3b7adab9..29952da5 100644 --- a/backend/src/core/auth/security.py +++ b/backend/src/core/auth/security.py @@ -8,6 +8,7 @@ import bcrypt + # #region verify_password [TYPE Function] # @BRIEF Verifies a plain password against a hashed password. # @PRE: plain_password is a string, hashed_password is a bcrypt hash. diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py index a4311eb7..b7f8a95c 100644 --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -16,13 +16,17 @@ import json import os from pathlib import Path -from typing import Any, Optional, List +from typing import Any + from sqlalchemy.orm import Session -from .config_models import AppConfig, Environment, GlobalSettings -from .database import SessionLocal + from ..models.config import AppConfigRecord from ..models.mapping import Environment as EnvironmentRecord -from .logger import logger, configure_logger, belief_scope +from .config_models import AppConfig, Environment, GlobalSettings +from .database import SessionLocal +from .logger import belief_scope, configure_logger, logger + + # #region ConfigManager [C:5] [TYPE Class] # @BRIEF Handles application configuration load, validation, mutation, and persistence lifecycle. # @PRE: Database is accessible and AppConfigRecord schema is loaded. @@ -106,7 +110,7 @@ class ConfigManager: "has_settings": "settings" in merged_payload, "extra_sections": sorted( key - for key in merged_payload.keys() + for key in merged_payload if key not in {"environments", "settings"} ), }, @@ -145,7 +149,7 @@ class ConfigManager: # #endregion _load_from_legacy_file # #region _get_record [TYPE Function] # @PURPOSE: Resolve global configuration record from DB. - def _get_record(self, session: Session) -> Optional[AppConfigRecord]: + def _get_record(self, session: Session) -> AppConfigRecord | None: with belief_scope("ConfigManager._get_record"): record = ( session.query(AppConfigRecord) @@ -303,7 +307,7 @@ class ConfigManager: # #region _save_config_to_db [TYPE Function] # @PURPOSE: Persist provided AppConfig into the global DB configuration record. def _save_config_to_db( - self, config: AppConfig, session: Optional[Session] = None + self, config: AppConfig, session: Session | None = None ) -> None: with belief_scope("ConfigManager._save_config_to_db"): owns_session = session is None @@ -426,7 +430,7 @@ class ConfigManager: # #endregion validate_path # #region get_environments [TYPE Function] # @PURPOSE: Return all configured environments. - def get_environments(self) -> List[Environment]: + def get_environments(self) -> list[Environment]: with belief_scope("ConfigManager.get_environments"): return list(self.config.environments) # #endregion get_environments @@ -438,7 +442,7 @@ class ConfigManager: # #endregion has_environments # #region get_environment [TYPE Function] # @PURPOSE: Resolve a configured environment by identifier. - def get_environment(self, env_id: str) -> Optional[Environment]: + def get_environment(self, env_id: str) -> Environment | None: with belief_scope("ConfigManager.get_environment", f"env_id={env_id}"): normalized = str(env_id or "").strip() if not normalized: @@ -534,4 +538,4 @@ class ConfigManager: return True # #endregion delete_environment # #endregion ConfigManager -# #endregion ConfigManager \ No newline at end of file +# #endregion ConfigManager diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 0d0b4b08..f498cf2e 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -4,8 +4,9 @@ # @RELATION IMPLEMENTS -> [CoreContracts] # @RELATION IMPLEMENTS -> [ConnectionContracts] + from pydantic import BaseModel, Field -from typing import List, Optional + from ..models.storage import StorageConfig from ..services.llm_prompt_templates import ( DEFAULT_LLM_ASSISTANT_SETTINGS, @@ -50,7 +51,7 @@ class LoggingConfig(BaseModel): task_log_level: str = ( "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR) ) - file_path: Optional[str] = None + file_path: str | None = None max_bytes: int = 10 * 1024 * 1024 backup_count: int = 5 enable_belief_state: bool = True @@ -62,8 +63,8 @@ class LoggingConfig(BaseModel): # #region CleanReleaseConfig [TYPE DataClass] # @BRIEF Configuration for clean release compliance subsystem. class CleanReleaseConfig(BaseModel): - active_policy_id: Optional[str] = None - active_registry_id: Optional[str] = None + active_policy_id: str | None = None + active_registry_id: str | None = None # #endregion CleanReleaseConfig @@ -86,10 +87,10 @@ class FeaturesConfig(BaseModel): class GlobalSettings(BaseModel): storage: StorageConfig = Field(default_factory=StorageConfig) clean_release: CleanReleaseConfig = Field(default_factory=CleanReleaseConfig) - default_environment_id: Optional[str] = None + default_environment_id: str | None = None logging: LoggingConfig = Field(default_factory=LoggingConfig) features: FeaturesConfig = Field(default_factory=FeaturesConfig) - connections: List[dict] = [] + connections: list[dict] = [] llm: dict = Field( default_factory=lambda: { "providers": [], @@ -120,7 +121,7 @@ class GlobalSettings(BaseModel): # #region AppConfig [TYPE DataClass] # @BRIEF The root configuration model containing all application settings. class AppConfig(BaseModel): - environments: List[Environment] = [] + environments: list[Environment] = [] settings: GlobalSettings diff --git a/backend/src/core/cot_logger.py b/backend/src/core/cot_logger.py index 667180cf..0e51d231 100644 --- a/backend/src/core/cot_logger.py +++ b/backend/src/core/cot_logger.py @@ -10,12 +10,10 @@ # @SIDE_EFFECT: Writes structured JSON to the 'cot' Python logger. # @DATA_CONTRACT: Log call -> Single-line JSON to logging.StreamHandler/file. -import json import logging import uuid from contextvars import ContextVar -from datetime import datetime, timezone -from typing import Any, Dict, Optional +from typing import Any # #region cot_trace_context [C:1] [TYPE Data] [SEMANTICS contextvar, trace_id, span_id, propagation] # @BRIEF ContextVars for trace ID and span ID propagation across async boundaries. @@ -29,13 +27,13 @@ cot_logger = logging.getLogger("cot") # #endregion cot_logger_instance __all__ = [ + "MarkerLogger", + "get_trace_id", + "log", + "pop_span", + "push_span", "seed_trace_id", "set_trace_id", - "get_trace_id", - "push_span", - "pop_span", - "log", - "MarkerLogger", ] # #region seed_trace_id [C:1] [TYPE Function] [SEMANTICS trace_id, uuid, contextvar, set] @@ -119,9 +117,9 @@ def log( src: str, marker: str, intent: str, - payload: Optional[Dict[str, Any]] = None, - error: Optional[str] = None, - level: Optional[str] = None, + payload: dict[str, Any] | None = None, + error: str | None = None, + level: str | None = None, ) -> None: """Emit a single-line structured JSON log record through the 'cot' logger. @@ -143,7 +141,7 @@ def log( # Build structured extra data for the LogRecord. # CotJsonFormatter will read these attributes to produce the final JSON output, # consistent with the main superset_tools_app logger. - extra: Dict[str, Any] = { + extra: dict[str, Any] = { "marker": marker, "intent": intent, "src": src, @@ -196,7 +194,7 @@ class MarkerLogger: # #region MarkerLogger.reason [TYPE Function] # @BRIEF Log a REASON marker — strict deduction, core logic. def reason( - self, intent: str, *, payload: Optional[Dict[str, Any]] = None + self, intent: str, *, payload: dict[str, Any] | None = None ) -> None: """Log a REASON marker (INFO level).""" log(self._module_name, "REASON", intent, payload=payload) @@ -206,7 +204,7 @@ class MarkerLogger: # #region MarkerLogger.reflect [TYPE Function] # @BRIEF Log a REFLECT marker — self-check, structural validation. def reflect( - self, intent: str, *, payload: Optional[Dict[str, Any]] = None + self, intent: str, *, payload: dict[str, Any] | None = None ) -> None: """Log a REFLECT marker (INFO level).""" log(self._module_name, "REFLECT", intent, payload=payload) @@ -219,8 +217,8 @@ class MarkerLogger: self, intent: str, *, - payload: Optional[Dict[str, Any]] = None, - error: Optional[str] = None, + payload: dict[str, Any] | None = None, + error: str | None = None, ) -> None: """Log an EXPLORE marker (WARNING level). diff --git a/backend/src/core/database.py b/backend/src/core/database.py index c3ae7bd3..a564ea3b 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -8,25 +8,27 @@ # # @INVARIANT: A single engine instance is used for the entire application. +import os +from pathlib import Path + from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import sessionmaker -from ..models.mapping import Base -from ..models.connection import ConnectionConfig + +from ..models import assistant as _assistant_models # noqa: F401 +from ..models import auth as _auth_models # noqa: F401 +from ..models import clean_release as _clean_release_models # noqa: F401 +from ..models import config as _config_models # noqa: F401 +from ..models import connection as _connection_models # noqa: F401 +from ..models import dataset_review as _dataset_review_models # noqa: F401 +from ..models import llm as _llm_models # noqa: F401 +from ..models import profile as _profile_models # noqa: F401 # Import models to ensure they're registered with Base from ..models import task as _task_models # noqa: F401 -from ..models import auth as _auth_models # noqa: F401 -from ..models import config as _config_models # noqa: F401 -from ..models import llm as _llm_models # noqa: F401 -from ..models import assistant as _assistant_models # noqa: F401 -from ..models import profile as _profile_models # noqa: F401 -from ..models import clean_release as _clean_release_models # noqa: F401 -from ..models import connection as _connection_models # noqa: F401 -from ..models import dataset_review as _dataset_review_models # noqa: F401 -from .logger import belief_scope, logger +from ..models.connection import ConnectionConfig +from ..models.mapping import Base from .auth.config import auth_config -import os -from pathlib import Path +from .logger import belief_scope, logger # #region BASE_DIR [C:1] [TYPE Variable] # @BRIEF Base directory for the backend. diff --git a/backend/src/core/encryption_key.py b/backend/src/core/encryption_key.py index bc677128..9fccfde0 100644 --- a/backend/src/core/encryption_key.py +++ b/backend/src/core/encryption_key.py @@ -15,7 +15,7 @@ from pathlib import Path from cryptography.fernet import Fernet -from .logger import logger, belief_scope +from .logger import belief_scope, logger DEFAULT_ENV_FILE_PATH = Path(__file__).resolve().parents[2] / ".env" diff --git a/backend/src/core/logger.py b/backend/src/core/logger.py index 72b771c3..0bb0eca8 100755 --- a/backend/src/core/logger.py +++ b/backend/src/core/logger.py @@ -15,16 +15,16 @@ import json import logging import threading import types -from datetime import datetime, timezone from contextlib import contextmanager +from datetime import UTC, datetime from logging.handlers import RotatingFileHandler +from typing import Any from pydantic import BaseModel, Field -from typing import Dict, Any, List, Optional -from collections import deque -from .cot_logger import _trace_id, _span_id, log as cot_log -from .ws_log_handler import WebSocketLogHandler # noqa: F401 +from .cot_logger import _span_id, _trace_id +from .cot_logger import log as cot_log +from .ws_log_handler import WebSocketLogHandler # Thread-local storage for belief state _belief_state = threading.local() @@ -79,7 +79,7 @@ class CotJsonFormatter(logging.Formatter): intent = record.getMessage() log_obj = { - "ts": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "ts": datetime.now(UTC).isoformat(timespec="milliseconds"), "level": record.levelname, "trace_id": _trace_id.get() or "no-trace", "src": src, @@ -128,7 +128,7 @@ class LogEntry(BaseModel): timestamp: datetime = Field(default_factory=datetime.utcnow) level: str message: str - context: Optional[Dict[str, Any]] = None + context: dict[str, Any] | None = None # #endregion LogEntry @@ -151,7 +151,7 @@ def belief_scope(anchor_id: str, message: str = ""): cot_log(anchor_id, "REFLECT", "Coherence OK", level="DEBUG") except Exception as e: # Log exception as EXPLORE (always logged) - cot_log(anchor_id, "EXPLORE", f"Assumption violated", error=str(e), level="WARNING") + cot_log(anchor_id, "EXPLORE", "Assumption violated", error=str(e), level="WARNING") raise finally: _belief_state.anchor_id = old_anchor diff --git a/backend/src/core/logger/__tests__/test_logger.py b/backend/src/core/logger/__tests__/test_logger.py index c7944955..054cbae4 100644 --- a/backend/src/core/logger/__tests__/test_logger.py +++ b/backend/src/core/logger/__tests__/test_logger.py @@ -4,18 +4,17 @@ # @RELATION VERIFIES -> src.core.logger import sys from pathlib import Path + # Add src to path sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src")) -import pytest import logging -from src.core.logger import ( - belief_scope, - logger, - configure_logger, - get_task_log_level, - should_log_task_level -) + +import pytest + from src.core.config_models import LoggingConfig +from src.core.logger import belief_scope, configure_logger, get_task_log_level, logger, should_log_task_level + + @pytest.fixture(autouse=True) def reset_logger_state(): """Reset logger state before each test to avoid cross-test contamination.""" @@ -49,7 +48,7 @@ def test_belief_scope_logs_reason_reflect_at_debug(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("TestFunction"): logger.info("Doing something important") @@ -58,7 +57,7 @@ def test_belief_scope_logs_reason_reflect_at_debug(caplog): assert any("[REASON]" in msg and "TestFunction" in msg for msg in log_messages), "REASON log not found" assert any("Doing something important" in msg for msg in log_messages), "Inline info log not found" assert any("[REFLECT]" in msg and "TestFunction" in msg for msg in log_messages), "REFLECT log not found" - + # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) @@ -77,17 +76,16 @@ def test_belief_scope_error_handling(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") - with pytest.raises(ValueError): - with belief_scope("FailingFunction"): - raise ValueError("Something went wrong") + with pytest.raises(ValueError), belief_scope("FailingFunction"): + raise ValueError("Something went wrong") log_messages = [record.message for record in caplog.records] assert any("[REASON]" in msg and "FailingFunction" in msg for msg in log_messages), "REASON log not found" assert any("[EXPLORE]" in msg and "FailingFunction" in msg and "Something went wrong" in msg for msg in log_messages), "EXPLORE log not found" # REFLECT should not be logged on failure - + # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) @@ -106,13 +104,13 @@ def test_belief_scope_success_reflect(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("SuccessFunction"): pass log_messages = [record.message for record in caplog.records] assert any("[REFLECT]" in msg and "SuccessFunction" in msg for msg in log_messages), "REFLECT log not found" - + # #endregion test_belief_scope_success_reflect # #region test_belief_scope_not_visible_at_info [TYPE Function] # @RELATION BINDS_TO -> test_logger @@ -168,7 +166,7 @@ def test_configure_logger_task_log_level(): enable_belief_state=True ) configure_logger(config) - + assert get_task_log_level() == "DEBUG", "task_log_level should be DEBUG" assert should_log_task_level("DEBUG") is True, "DEBUG should be logged at DEBUG threshold" # #endregion test_configure_logger_task_log_level @@ -186,14 +184,14 @@ def test_enable_belief_state_flag(caplog): enable_belief_state=False ) configure_logger(config) - + caplog.set_level("DEBUG") - + with belief_scope("DisabledFunction"): logger.info("Doing something") - + log_messages = [record.message for record in caplog.records] - + # "Entering DisabledFunction" (explicit logger.reason()) should NOT be logged when disabled assert not any("Entering" in msg for msg in log_messages), "REASON entry should not be logged when disabled" # REFLECT should still be logged (internal tracking, not gated by _enable_belief_state) @@ -205,6 +203,7 @@ def test_enable_belief_state_flag(caplog): def test_belief_scope_missing_anchor(): """Test that belief_scope enforces anchor_id to be provided.""" import pytest + from src.core.logger import belief_scope with pytest.raises(TypeError): # Missing required positional argument 'anchor_id' @@ -218,9 +217,10 @@ def test_configure_logger_post_conditions(tmp_path): """Test that configure_logger satisfies all @POST conditions.""" import logging from logging.handlers import RotatingFileHandler - from src.core.config_models import LoggingConfig - from src.core.logger import configure_logger, logger, BeliefFormatter, get_task_log_level + import src.core.logger as logger_module + from src.core.config_models import LoggingConfig + from src.core.logger import BeliefFormatter, configure_logger, get_task_log_level, logger log_file = tmp_path / "test.log" config = LoggingConfig( level="WARNING", @@ -228,24 +228,24 @@ def test_configure_logger_post_conditions(tmp_path): enable_belief_state=False, file_path=str(log_file) ) - + configure_logger(config) - + # 1. Logger level is updated assert logger.level == logging.WARNING - + # 2. Handlers are updated (file handler removed old ones, added new one) file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)] assert len(file_handlers) == 1 import pathlib assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve() - + # 3. Formatter is set to BeliefFormatter for handler in logger.handlers: assert isinstance(handler.formatter, BeliefFormatter) - + # 4. Global states - assert getattr(logger_module, '_enable_belief_state') is False + assert logger_module._enable_belief_state is False assert get_task_log_level() == "DEBUG" # #endregion test_configure_logger_post_conditions -# #endregion test_logger \ No newline at end of file +# #endregion test_logger diff --git a/backend/src/core/mapping_service.py b/backend/src/core/mapping_service.py index 2d18481c..7cbc6193 100644 --- a/backend/src/core/mapping_service.py +++ b/backend/src/core/mapping_service.py @@ -11,14 +11,17 @@ # @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]} # # @INVARIANT: sync_environment must handle remote API failures gracefully. -from typing import Dict, List, Optional -from datetime import datetime, timezone -from sqlalchemy.orm import Session +from datetime import UTC, datetime + from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger -from src.models.mapping import ResourceMapping, ResourceType -from src.core.logger import logger, belief_scope +from sqlalchemy.orm import Session + from src.core.cot_logger import seed_trace_id +from src.core.logger import belief_scope, logger +from src.models.mapping import ResourceMapping, ResourceType + + # #region IdMappingService [C:5] [TYPE Class] # @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs. # @PRE: db_session is an active SQLAlchemy Session bound to mapping tables. @@ -57,7 +60,7 @@ class IdMappingService: # @PARAM: environments (List[str]) - List of environment IDs to sync. # @PARAM: superset_client_factory - Function to get a client for an environment. def start_scheduler( - self, cron_string: str, environments: List[str], superset_client_factory + self, cron_string: str, environments: list[str], superset_client_factory ): with belief_scope("IdMappingService.start_scheduler"): if self._sync_job: @@ -173,7 +176,7 @@ class IdMappingService: if mapping: mapping.remote_integer_id = res_id mapping.resource_name = res_name - mapping.last_synced_at = datetime.now(timezone.utc) + mapping.last_synced_at = datetime.now(UTC) else: new_mapping = ResourceMapping( environment_id=environment_id, @@ -181,7 +184,7 @@ class IdMappingService: uuid=res_uuid, remote_integer_id=res_id, resource_name=res_name, - last_synced_at=datetime.now(timezone.utc), + last_synced_at=datetime.now(UTC), ) self.db.add(new_mapping) total_synced += 1 @@ -228,7 +231,7 @@ class IdMappingService: # @RETURN: Optional[int] def get_remote_id( self, environment_id: str, resource_type: ResourceType, uuid: str - ) -> Optional[int]: + ) -> int | None: mapping = ( self.db.query(ResourceMapping) .filter_by( @@ -250,8 +253,8 @@ class IdMappingService: # @PARAM: uuids (List[str]) # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID def get_remote_ids_batch( - self, environment_id: str, resource_type: ResourceType, uuids: List[str] - ) -> Dict[str, int]: + self, environment_id: str, resource_type: ResourceType, uuids: list[str] + ) -> dict[str, int]: if not uuids: return {} mappings = ( @@ -272,4 +275,4 @@ class IdMappingService: return result # #endregion get_remote_ids_batch # #endregion IdMappingService -# #endregion IdMappingServiceModule \ No newline at end of file +# #endregion IdMappingServiceModule diff --git a/backend/src/core/middleware/trace.py b/backend/src/core/middleware/trace.py index 5ee164e1..708ae4dc 100644 --- a/backend/src/core/middleware/trace.py +++ b/backend/src/core/middleware/trace.py @@ -16,6 +16,7 @@ from starlette.types import ASGIApp from ..cot_logger import seed_trace_id, set_trace_id + # #region TraceContextMiddleware [C:2] [TYPE Class] [SEMANTICS middleware, trace, context, dispatch] # @BRIEF Starlette BaseHTTPMiddleware that seeds a trace_id per request. class TraceContextMiddleware(BaseHTTPMiddleware): diff --git a/backend/src/core/migration/__init__.py b/backend/src/core/migration/__init__.py index 9fa5ad5c..52848a2a 100644 --- a/backend/src/core/migration/__init__.py +++ b/backend/src/core/migration/__init__.py @@ -1,8 +1,8 @@ # #region MigrationPackage [TYPE Module] [SEMANTICS migration, package, init] -from .dry_run_orchestrator import MigrationDryRunService from .archive_parser import MigrationArchiveParser +from .dry_run_orchestrator import MigrationDryRunService -__all__ = ["MigrationDryRunService", "MigrationArchiveParser"] +__all__ = ["MigrationArchiveParser", "MigrationDryRunService"] # #endregion MigrationPackage diff --git a/backend/src/core/migration/archive_parser.py b/backend/src/core/migration/archive_parser.py index e81c92dc..d0b0493c 100644 --- a/backend/src/core/migration/archive_parser.py +++ b/backend/src/core/migration/archive_parser.py @@ -7,9 +7,13 @@ import json import tempfile import zipfile from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any + import yaml -from ..logger import logger, belief_scope + +from ..logger import belief_scope, logger + + # #region MigrationArchiveParser [TYPE Class] # @BRIEF Extract normalized dashboards/charts/datasets metadata from ZIP archives. # @RELATION CONTAINS -> [extract_objects_from_zip] @@ -24,9 +28,9 @@ class MigrationArchiveParser: # @RETURN: Dict[str, List[Dict[str, Any]]] def extract_objects_from_zip( self, zip_path: str - ) -> Dict[str, List[Dict[str, Any]]]: + ) -> dict[str, list[dict[str, Any]]]: with belief_scope("MigrationArchiveParser.extract_objects_from_zip"): - result: Dict[str, List[Dict[str, Any]]] = { + result: dict[str, list[dict[str, Any]]] = { "dashboards": [], "charts": [], "datasets": [], @@ -49,15 +53,15 @@ class MigrationArchiveParser: # @POST: Returns only valid normalized objects. def _collect_yaml_objects( self, root_dir: Path, object_type: str - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: with belief_scope("MigrationArchiveParser._collect_yaml_objects"): files = list(root_dir.glob(f"**/{object_type}/**/*.yaml")) + list( root_dir.glob(f"**/{object_type}/*.yaml") ) - objects: List[Dict[str, Any]] = [] + objects: list[dict[str, Any]] = [] for file_path in set(files): try: - with open(file_path, "r") as file_obj: + with open(file_path) as file_obj: payload = yaml.safe_load(file_obj) or {} normalized = self._normalize_object_payload(payload, object_type) if normalized: @@ -74,8 +78,8 @@ class MigrationArchiveParser: # @PRE: payload is parsed YAML mapping. # @POST: Returns normalized descriptor with `uuid`, `title`, and `signature`. def _normalize_object_payload( - self, payload: Dict[str, Any], object_type: str - ) -> Optional[Dict[str, Any]]: + self, payload: dict[str, Any], object_type: str + ) -> dict[str, Any] | None: with belief_scope("MigrationArchiveParser._normalize_object_payload"): if not isinstance(payload, dict): return None @@ -134,4 +138,4 @@ class MigrationArchiveParser: return None # #endregion _normalize_object_payload # #endregion MigrationArchiveParser -# #endregion MigrationArchiveParserModule \ No newline at end of file +# #endregion MigrationArchiveParserModule diff --git a/backend/src/core/migration/dry_run_orchestrator.py b/backend/src/core/migration/dry_run_orchestrator.py index 002b3522..f7039fc9 100644 --- a/backend/src/core/migration/dry_run_orchestrator.py +++ b/backend/src/core/migration/dry_run_orchestrator.py @@ -6,18 +6,22 @@ # @RELATION DEPENDS_ON -> [MigrationArchiveParser] # @RELATION DEPENDS_ON -> [RiskAssessorModule] # @INVARIANT: Dry run is informative only and must not mutate target environment. -from datetime import datetime, timezone import json -from typing import Any, Dict, List +from datetime import UTC, datetime +from typing import Any + from sqlalchemy.orm import Session + from ...models.dashboard import DashboardSelection from ...models.mapping import DatabaseMapping -from ..logger import logger, belief_scope -from .archive_parser import MigrationArchiveParser -from .risk_assessor import build_risks, score_risks +from ..logger import belief_scope, logger from ..migration_engine import MigrationEngine from ..superset_client import SupersetClient from ..utils.fileio import create_temp_file +from .archive_parser import MigrationArchiveParser +from .risk_assessor import build_risks, score_risks + + # #region MigrationDryRunService [TYPE Class] # @BRIEF Build deterministic diff/risk payload for migration pre-flight. # @RELATION CONTAINS -> [__init__] @@ -52,7 +56,7 @@ class MigrationDryRunService: source_client: SupersetClient, target_client: SupersetClient, db: Session, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("MigrationDryRunService.run"): logger.reason( "Starting dry-run pipeline", @@ -75,24 +79,23 @@ class MigrationDryRunService: exported_content, _ = source_client.export_dashboard(int(dashboard_id)) with create_temp_file( content=exported_content, suffix=".zip" - ) as source_zip: - with create_temp_file(suffix=".zip") as transformed_zip: - success = engine.transform_zip( - str(source_zip), - str(transformed_zip), - db_mapping, - strip_databases=False, - target_env_id=selection.target_env_id, - fix_cross_filters=selection.fix_cross_filters, + ) as source_zip, create_temp_file(suffix=".zip") as transformed_zip: + success = engine.transform_zip( + str(source_zip), + str(transformed_zip), + db_mapping, + strip_databases=False, + target_env_id=selection.target_env_id, + fix_cross_filters=selection.fix_cross_filters, + ) + if not success: + raise ValueError( + f"Failed to transform export archive for dashboard {dashboard_id}" ) - if not success: - raise ValueError( - f"Failed to transform export archive for dashboard {dashboard_id}" - ) - extracted = self.parser.extract_objects_from_zip( - str(transformed_zip) - ) - self._accumulate_objects(transformed, extracted) + extracted = self.parser.extract_objects_from_zip( + str(transformed_zip) + ) + self._accumulate_objects(transformed, extracted) source_objects = { key: list(value.values()) for key, value in transformed.items() } @@ -136,7 +139,7 @@ class MigrationDryRunService: extra={"src": "MigrationDryRunService.run"}, ) return { - "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at": datetime.now(UTC).isoformat(), "selection": selection.model_dump(), "selected_dashboard_titles": selected_titles, "diff": diff, @@ -148,7 +151,7 @@ class MigrationDryRunService: # @PURPOSE: Resolve UUID mapping for optional DB config replacement. def _load_db_mapping( self, db: Session, selection: DashboardSelection - ) -> Dict[str, str]: + ) -> dict[str, str]: rows = ( db.query(DatabaseMapping) .filter( @@ -163,8 +166,8 @@ class MigrationDryRunService: # @PURPOSE: Merge extracted resources by UUID to avoid duplicates. def _accumulate_objects( self, - target: Dict[str, Dict[str, Dict[str, Any]]], - source: Dict[str, List[Dict[str, Any]]], + target: dict[str, dict[str, dict[str, Any]]], + source: dict[str, list[dict[str, Any]]], ) -> None: for object_type in ("dashboards", "charts", "datasets"): for item in source.get(object_type, []): @@ -175,9 +178,9 @@ class MigrationDryRunService: # #region _index_by_uuid [TYPE Function] # @PURPOSE: Build UUID-index map for normalized resources. def _index_by_uuid( - self, objects: List[Dict[str, Any]] - ) -> Dict[str, Dict[str, Any]]: - indexed: Dict[str, Dict[str, Any]] = {} + self, objects: list[dict[str, Any]] + ) -> dict[str, dict[str, Any]]: + indexed: dict[str, dict[str, Any]] = {} for obj in objects: uuid = obj.get("uuid") if uuid: @@ -188,12 +191,12 @@ class MigrationDryRunService: # @PURPOSE: Compute create/update/delete buckets by UUID+signature. # @RELATION: DEPENDS_ON -> [_index_by_uuid] def _build_object_diff( - self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]] - ) -> Dict[str, List[Dict[str, Any]]]: + self, source_objects: list[dict[str, Any]], target_objects: list[dict[str, Any]] + ) -> dict[str, list[dict[str, Any]]]: target_index = self._index_by_uuid(target_objects) - created: List[Dict[str, Any]] = [] - updated: List[Dict[str, Any]] = [] - deleted: List[Dict[str, Any]] = [] + created: list[dict[str, Any]] = [] + updated: list[dict[str, Any]] = [] + deleted: list[dict[str, Any]] = [] for source_obj in source_objects: source_uuid = str(source_obj.get("uuid")) target_obj = target_index.get(source_uuid) @@ -214,7 +217,7 @@ class MigrationDryRunService: # @PURPOSE: Pull target metadata and normalize it into comparable signatures. def _build_target_signatures( self, client: SupersetClient - ) -> Dict[str, List[Dict[str, Any]]]: + ) -> dict[str, list[dict[str, Any]]]: _, dashboards = client.get_dashboards( query={ "columns": [ @@ -325,12 +328,12 @@ class MigrationDryRunService: # @PURPOSE: Build risk items for missing datasource, broken refs, overwrite, owner mismatch. def _build_risks( self, - source_objects: Dict[str, List[Dict[str, Any]]], - target_objects: Dict[str, List[Dict[str, Any]]], - diff: Dict[str, Dict[str, List[Dict[str, Any]]]], + source_objects: dict[str, list[dict[str, Any]]], + target_objects: dict[str, list[dict[str, Any]]], + diff: dict[str, dict[str, list[dict[str, Any]]]], target_client: SupersetClient, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: return build_risks(source_objects, target_objects, diff, target_client) # #endregion _build_risks # #endregion MigrationDryRunService -# #endregion MigrationDryRunOrchestratorModule \ No newline at end of file +# #endregion MigrationDryRunOrchestratorModule diff --git a/backend/src/core/migration/risk_assessor.py b/backend/src/core/migration/risk_assessor.py index 1106c0ec..01e56ac6 100644 --- a/backend/src/core/migration/risk_assessor.py +++ b/backend/src/core/migration/risk_assessor.py @@ -24,9 +24,9 @@ # @UX_RECOVERY: [N/A] -> [Caller-level retry/recovery] # @UX_REACTIVITY: [N/A] -> [Backend synchronous function contracts] -from typing import Any, Dict, List +from typing import Any -from ..logger import logger, belief_scope +from ..logger import belief_scope, logger from ..superset_client import SupersetClient @@ -36,10 +36,10 @@ from ..superset_client import SupersetClient # @POST: Returns mapping keyed by string uuid; only truthy uuid values are included. # @SIDE_EFFECT: Emits reasoning/reflective logs only. # @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]] -def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: +def index_by_uuid(objects: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: with belief_scope("risk_assessor.index_by_uuid"): logger.reason("Building UUID index", extra={"objects_count": len(objects)}) - indexed: Dict[str, Dict[str, Any]] = {} + indexed: dict[str, dict[str, Any]] = {} for obj in objects: uuid = obj.get("uuid") if uuid: @@ -57,13 +57,13 @@ def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: # @POST: Returns sorted unique owner identifiers as strings. # @SIDE_EFFECT: Emits reasoning/reflective logs only. # @DATA_CONTRACT: Any -> List[str] -def extract_owner_identifiers(owners: Any) -> List[str]: +def extract_owner_identifiers(owners: Any) -> list[str]: with belief_scope("risk_assessor.extract_owner_identifiers"): logger.reason("Normalizing owner identifiers") if not isinstance(owners, list): logger.reflect("Owners payload is not list; returning empty identifiers") return [] - ids: List[str] = [] + ids: list[str] = [] for owner in owners: if isinstance(owner, dict): if owner.get("username"): @@ -97,14 +97,14 @@ def extract_owner_identifiers(owners: Any) -> List[str]: # @DATA_CONTRACT: SupersetClient # @DATA_CONTRACT: ) -> List[Dict[str, Any]] def build_risks( - source_objects: Dict[str, List[Dict[str, Any]]], - target_objects: Dict[str, List[Dict[str, Any]]], - diff: Dict[str, Dict[str, List[Dict[str, Any]]]], + source_objects: dict[str, list[dict[str, Any]]], + target_objects: dict[str, list[dict[str, Any]]], + diff: dict[str, dict[str, list[dict[str, Any]]]], target_client: SupersetClient, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: with belief_scope("risk_assessor.build_risks"): logger.reason("Building migration risks from diff payload") - risks: List[Dict[str, Any]] = [] + risks: list[dict[str, Any]] = [] for object_type in ("dashboards", "charts", "datasets"): for item in diff[object_type]["update"]: risks.append( @@ -181,7 +181,7 @@ def build_risks( # @POST: Returns dict with score in [0,100], derived level, and original items. # @SIDE_EFFECT: Emits reasoning/reflective logs only. # @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any] -def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]: +def score_risks(risk_items: list[dict[str, Any]]) -> dict[str, Any]: with belief_scope("risk_assessor.score_risks"): logger.reason("Scoring risk items", extra={"risk_items_count": len(risk_items)}) weights = {"high": 25, "medium": 10, "low": 5} diff --git a/backend/src/core/migration_engine.py b/backend/src/core/migration_engine.py index a38379fa..0c250583 100644 --- a/backend/src/core/migration_engine.py +++ b/backend/src/core/migration_engine.py @@ -11,17 +11,21 @@ # @SIDE_EFFECT: Reads and writes temporary archive contents during transformation workflows and emits structured belief-state logs. # @DATA_CONTRACT: Input[zip_path, output_path, db_mapping, target_env_id?, fix_cross_filters?] -> Output[Transformed Superset archive] # @INVARIANT: ZIP structure and non-targeted metadata must remain valid after transformation. -import zipfile -import yaml -import os import json +import os import re import tempfile +import zipfile from pathlib import Path -from typing import Dict, Optional, List -from .logger import logger, belief_scope + +import yaml + from src.core.mapping_service import IdMappingService from src.models.mapping import ResourceType + +from .logger import belief_scope, logger + + # #region MigrationEngine [TYPE Class] # @BRIEF Engine for transforming Superset export ZIPs. # @RELATION CONTAINS -> [__init__, transform_zip, _transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata] @@ -33,7 +37,7 @@ class MigrationEngine: # @SIDE_EFFECT: Mutates in-memory engine state by storing dependency reference. # @DATA_CONTRACT: Input[Optional[IdMappingService]] -> Output[MigrationEngine] # @PARAM: mapping_service (Optional[IdMappingService]) - Used for resolving target environment integer IDs. - def __init__(self, mapping_service: Optional[IdMappingService] = None): + def __init__(self, mapping_service: IdMappingService | None = None): with belief_scope("MigrationEngine.__init__"): logger.reason("Initializing MigrationEngine") self.mapping_service = mapping_service @@ -57,9 +61,9 @@ class MigrationEngine: self, zip_path: str, output_path: str, - db_mapping: Dict[str, str], + db_mapping: dict[str, str], strip_databases: bool = True, - target_env_id: Optional[str] = None, + target_env_id: str | None = None, fix_cross_filters: bool = False, ) -> bool: """ @@ -133,12 +137,12 @@ class MigrationEngine: # @POST: database_uuid is replaced in-place only when source UUID is present in db_mapping. # @SIDE_EFFECT: Reads and conditionally rewrites YAML file on disk. # @DATA_CONTRACT: Input[(Path file_path, Dict[str,str] db_mapping)] -> Output[None] - def _transform_yaml(self, file_path: Path, db_mapping: Dict[str, str]): + def _transform_yaml(self, file_path: Path, db_mapping: dict[str, str]): with belief_scope("MigrationEngine._transform_yaml"): if not file_path.exists(): logger.explore(f"YAML file not found: {file_path}") raise FileNotFoundError(str(file_path)) - with open(file_path, "r") as f: + with open(file_path) as f: data = yaml.safe_load(f) if not data: return @@ -158,7 +162,7 @@ class MigrationEngine: # @DATA_CONTRACT: Input[Path] -> Output[Dict[int,str]] # @PARAM: temp_dir (Path) - Root dir of unpacked archive. # @RETURN: Dict[int, str] - Mapping of source Integer ID to UUID. - def _extract_chart_uuids_from_archive(self, temp_dir: Path) -> Dict[int, str]: + def _extract_chart_uuids_from_archive(self, temp_dir: Path) -> dict[int, str]: with belief_scope("MigrationEngine._extract_chart_uuids_from_archive"): # Implementation Note: This is a placeholder for the logic that extracts # actual Source IDs. In a real scenario, this involves parsing chart YAMLs @@ -170,7 +174,7 @@ class MigrationEngine: ) for cf in set(chart_files): try: - with open(cf, "r") as f: + with open(cf) as f: cdata = yaml.safe_load(f) if cdata and "id" in cdata and "uuid" in cdata: mapping[cdata["id"]] = cdata["uuid"] @@ -188,13 +192,13 @@ class MigrationEngine: # @PARAM: target_env_id (str) # @PARAM: source_map (Dict[int, str]) def _patch_dashboard_metadata( - self, file_path: Path, target_env_id: str, source_map: Dict[int, str] + self, file_path: Path, target_env_id: str, source_map: dict[int, str] ): with belief_scope("MigrationEngine._patch_dashboard_metadata"): try: if not file_path.exists(): return - with open(file_path, "r") as f: + with open(file_path) as f: data = yaml.safe_load(f) if not data or "json_metadata" not in data: return @@ -255,4 +259,4 @@ class MigrationEngine: logger.explore(f"Metadata patch failed for {file_path.name}: {e}") # #endregion _patch_dashboard_metadata # #endregion MigrationEngine -# #endregion MigrationEngineModule \ No newline at end of file +# #endregion MigrationEngineModule diff --git a/backend/src/core/plugin_base.py b/backend/src/core/plugin_base.py index 59301778..ea4eb2f5 100755 --- a/backend/src/core/plugin_base.py +++ b/backend/src/core/plugin_base.py @@ -1,7 +1,11 @@ from abc import ABC, abstractmethod -from typing import Dict, Any, Optional -from .logger import belief_scope +from typing import Any + from pydantic import BaseModel, Field + +from .logger import belief_scope + + # #region PluginBase [TYPE Class] [SEMANTICS plugin, interface, base, abstract] # @BRIEF Defines the abstract base class that all plugins must implement to be recognized by the system. It enforces a common structure for plugin metadata and execution. # @LAYER: Core @@ -77,7 +81,7 @@ class PluginBase(ABC): # @PRE: Plugin instance exists. # @POST: Returns string route or None. # @RETURN: Optional[str] - Frontend route. - def ui_route(self) -> Optional[str]: + def ui_route(self) -> str | None: """ The frontend route for the plugin's UI. Returns None if the plugin does not have a dedicated UI page. @@ -91,7 +95,7 @@ class PluginBase(ABC): # @PRE: Plugin instance exists. # @POST: Returns dict schema. # @RETURN: Dict[str, Any] - JSON schema. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: """ Returns the JSON schema for the plugin's input parameters. This schema will be used to generate the frontend form. @@ -105,7 +109,7 @@ class PluginBase(ABC): # @PARAM: params (Dict[str, Any]) - Validated input parameters. # @PRE: params must be a dictionary. # @POST: Plugin execution is completed. - async def execute(self, params: Dict[str, Any]): + async def execute(self, params: dict[str, Any]): with belief_scope("execute"): pass """ @@ -125,6 +129,6 @@ class PluginConfig(BaseModel): name: str = Field(..., description="Human-readable name for the plugin") description: str = Field(..., description="Brief description of what the plugin does") version: str = Field(..., description="Version of the plugin") - ui_route: Optional[str] = Field(None, description="Frontend route for the plugin UI") - input_schema: Dict[str, Any] = Field(..., description="JSON schema for input parameters", alias="schema") -# #endregion PluginConfig \ No newline at end of file + ui_route: str | None = Field(None, description="Frontend route for the plugin UI") + input_schema: dict[str, Any] = Field(..., description="JSON schema for input parameters", alias="schema") +# #endregion PluginConfig diff --git a/backend/src/core/plugin_loader.py b/backend/src/core/plugin_loader.py index 6a97ffa5..23eb1b8c 100755 --- a/backend/src/core/plugin_loader.py +++ b/backend/src/core/plugin_loader.py @@ -1,9 +1,11 @@ import importlib.util import os -import sys # Added this line -from typing import Dict, List, Optional -from .plugin_base import PluginBase, PluginConfig +import sys # Added this line + from .logger import belief_scope +from .plugin_base import PluginBase, PluginConfig + + # #region PluginLoader [C:3] [TYPE Class] [SEMANTICS plugin, loader, dynamic, import] # @BRIEF Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface. # @LAYER: Core @@ -21,8 +23,8 @@ class PluginLoader: def __init__(self, plugin_dir: str): with belief_scope("__init__"): self.plugin_dir = plugin_dir - self._plugins: Dict[str, PluginBase] = {} - self._plugin_configs: Dict[str, PluginConfig] = {} + self._plugins: dict[str, PluginBase] = {} + self._plugin_configs: dict[str, PluginConfig] = {} self._load_plugins() # #endregion __init__ # #region _load_plugins [TYPE Function] @@ -44,7 +46,7 @@ class PluginLoader: sys.path.insert(0, plugin_parent_dir) for filename in os.listdir(self.plugin_dir): file_path = os.path.join(self.plugin_dir, filename) - + # Handle directory-based plugins (packages) if os.path.isdir(file_path): init_file = os.path.join(file_path, "__init__.py") @@ -69,7 +71,7 @@ class PluginLoader: """ # All runtime code is imported through the canonical `src` package root. package_name = f"src.plugins.{module_name}" - + # print(f"DEBUG: Loading plugin {module_name} as {package_name}") spec = importlib.util.spec_from_file_location(package_name, file_path) if spec is None or spec.loader is None: @@ -113,7 +115,7 @@ class PluginLoader: # Basic validation to ensure it's a dictionary if not isinstance(schema, dict): raise TypeError("get_schema() must return a dictionary.") - + plugin_config = PluginConfig( id=plugin_instance.id, name=plugin_instance.name, @@ -139,7 +141,7 @@ class PluginLoader: # @POST: Returns plugin instance or None. # @PARAM: plugin_id (str) - The unique identifier of the plugin. # @RETURN: Optional[PluginBase] - The plugin instance if found, otherwise None. - def get_plugin(self, plugin_id: str) -> Optional[PluginBase]: + def get_plugin(self, plugin_id: str) -> PluginBase | None: with belief_scope("get_plugin"): """ Returns a loaded plugin instance by its ID. @@ -151,7 +153,7 @@ class PluginLoader: # @PRE: None. # @POST: Returns list of all PluginConfig objects. # @RETURN: List[PluginConfig] - A list of plugin configurations. - def get_all_plugin_configs(self) -> List[PluginConfig]: + def get_all_plugin_configs(self) -> list[PluginConfig]: with belief_scope("get_all_plugin_configs"): """ Returns a list of all loaded plugin configurations. @@ -171,4 +173,4 @@ class PluginLoader: """ return plugin_id in self._plugins # #endregion has_plugin -# #endregion PluginLoader \ No newline at end of file +# #endregion PluginLoader diff --git a/backend/src/core/scheduler.py b/backend/src/core/scheduler.py index c8c506fc..c2274503 100644 --- a/backend/src/core/scheduler.py +++ b/backend/src/core/scheduler.py @@ -2,14 +2,18 @@ # @BRIEF Manages scheduled tasks using APScheduler. # @LAYER: Core # @RELATION DEPENDS_ON -> TaskManager +import asyncio +from datetime import date, datetime, time, timedelta + from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger -from .logger import logger, belief_scope -from .cot_logger import seed_trace_id + from .config_manager import ConfigManager +from .cot_logger import seed_trace_id from .database import SessionLocal -import asyncio -from datetime import datetime, time, timedelta, date +from .logger import belief_scope, logger + + # #region SchedulerService [C:3] [TYPE Class] [SEMANTICS scheduler, service, apscheduler] # @BRIEF Provides a service to manage scheduled backup tasks. # @RELATION DEPENDS_ON -> [ThrottledSchedulerConfigurator] @@ -124,6 +128,7 @@ class SchedulerService: f"schedule_id={schedule_id}, job_id={job_id}, cron={cron_expression}", ): from zoneinfo import ZoneInfo + from ..plugins.translate.scheduler import execute_scheduled_translation job_id_aps = f"translate_{schedule_id}" @@ -245,4 +250,4 @@ class ThrottledSchedulerConfigurator: return scheduled_times # #endregion calculate_schedule # #endregion ThrottledSchedulerConfigurator -# #endregion SchedulerModule \ No newline at end of file +# #endregion SchedulerModule diff --git a/backend/src/core/superset_client/__init__.py b/backend/src/core/superset_client/__init__.py index d3137b85..0ccf4c4b 100644 --- a/backend/src/core/superset_client/__init__.py +++ b/backend/src/core/superset_client/__init__.py @@ -17,15 +17,15 @@ # @REJECTED: Keeping a single 2145-line file — violates fractal limit INV_7. from ._base import SupersetClientBase -from ._user_projection import SupersetUserProjectionMixin -from ._dashboards_list import SupersetDashboardsListMixin -from ._dashboards_filters import SupersetDashboardsFiltersMixin -from ._dashboards_crud import SupersetDashboardsCrudMixin from ._charts import SupersetChartsMixin +from ._dashboards_crud import SupersetDashboardsCrudMixin +from ._dashboards_filters import SupersetDashboardsFiltersMixin +from ._dashboards_list import SupersetDashboardsListMixin +from ._databases import SupersetDatabasesMixin from ._datasets import SupersetDatasetsMixin from ._datasets_preview import SupersetDatasetsPreviewMixin from ._datasets_preview_filters import SupersetDatasetsPreviewFiltersMixin -from ._databases import SupersetDatabasesMixin +from ._user_projection import SupersetUserProjectionMixin # #region SupersetClient [C:3] [TYPE Class] diff --git a/backend/src/core/superset_client/_base.py b/backend/src/core/superset_client/_base.py index 7bfa6f7d..f786e11b 100644 --- a/backend/src/core/superset_client/_base.py +++ b/backend/src/core/superset_client/_base.py @@ -5,15 +5,18 @@ # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetAPIError] # @RELATION DEPENDS_ON -> [get_filename_from_headers] -import json import zipfile from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Optional, cast + from requests import Response -from ..logger import logger as app_logger, belief_scope -from ..utils.network import APIClient, SupersetAPIError -from ..utils.fileio import get_filename_from_headers + from ..config_models import Environment +from ..logger import belief_scope +from ..logger import logger as app_logger +from ..utils.fileio import get_filename_from_headers +from ..utils.network import APIClient, SupersetAPIError + app_logger = cast(Any, app_logger) # #region SupersetClientBase [C:3] [TYPE Class] # @BRIEF Base class providing Superset client initialization, auth, pagination, and import/export plumbing. @@ -53,7 +56,7 @@ class SupersetClientBase: # #region SupersetClientAuthenticate [TYPE Function] [C:3] # @PURPOSE: Authenticates the client using the configured credentials. # @RELATION: CALLS -> [APIClient] - def authenticate(self) -> Dict[str, str]: + def authenticate(self) -> dict[str, str]: with belief_scope("SupersetClientAuthenticate"): app_logger.reason( "Authenticating Superset client", @@ -79,7 +82,7 @@ class SupersetClientBase: # --- Pagination helpers --- # #region SupersetClientValidateQueryParams [TYPE Function] [C:1] # @PURPOSE: Ensures query parameters have default page and page_size. - def _validate_query_params(self, query: Optional[Dict]) -> Dict: + def _validate_query_params(self, query: dict | None) -> dict: with belief_scope("_validate_query_params"): # Superset list endpoints commonly cap page_size at 100. # Using 100 avoids partial fetches when larger values are silently truncated. @@ -100,7 +103,7 @@ class SupersetClientBase: # #region SupersetClientFetchAllPages [TYPE Function] [C:1] # @PURPOSE: Iterates through all pages to collect all data items. # @RELATION: CALLS -> [APIClient] - def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]: + def _fetch_all_pages(self, endpoint: str, pagination_options: dict) -> list[dict]: with belief_scope("_fetch_all_pages"): return self.network.fetch_paginated_data( endpoint=endpoint, pagination_options=pagination_options @@ -110,7 +113,7 @@ class SupersetClientBase: # #region SupersetClientDoImport [TYPE Function] [C:1] # @PURPOSE: Performs the actual multipart upload for import. # @RELATION: CALLS -> [APIClient] - def _do_import(self, file_name: Union[str, Path]) -> Dict: + def _do_import(self, file_name: str | Path) -> dict: with belief_scope("_do_import"): app_logger.debug(f"[_do_import][State] Uploading file: {file_name}") file_path = Path(file_name) @@ -159,7 +162,7 @@ class SupersetClientBase: # #endregion SupersetClientResolveExportFilename # #region SupersetClientValidateImportFile [TYPE Function] [C:1] # @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml. - def _validate_import_file(self, zip_path: Union[str, Path]) -> None: + def _validate_import_file(self, zip_path: str | Path) -> None: with belief_scope("_validate_import_file"): path = Path(zip_path) if not path.exists(): @@ -176,8 +179,8 @@ class SupersetClientBase: # @PURPOSE: Resolves a dashboard ID from either an ID or a slug. # @RELATION: CALLS -> [SupersetClientGetDashboards] def _resolve_target_id_for_delete( - self, dash_id: Optional[int], dash_slug: Optional[str] - ) -> Optional[int]: + self, dash_id: int | None, dash_slug: str | None + ) -> int | None: with belief_scope("_resolve_target_id_for_delete"): if dash_id is not None: return dash_id @@ -212,7 +215,7 @@ class SupersetClientBase: # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_all_resources( self, resource_type: str, since_dttm: Optional["datetime"] = None - ) -> List[Dict]: + ) -> list[dict]: with belief_scope( "SupersetClient.get_all_resources", f"type={resource_type}, since={since_dttm}", @@ -258,4 +261,4 @@ class SupersetClientBase: return data # #endregion SupersetClientGetAllResources # #endregion SupersetClientBase -# #endregion SupersetClientBase \ No newline at end of file +# #endregion SupersetClientBase diff --git a/backend/src/core/superset_client/_charts.py b/backend/src/core/superset_client/_charts.py index d2058cdd..48e148ea 100644 --- a/backend/src/core/superset_client/_charts.py +++ b/backend/src/core/superset_client/_charts.py @@ -2,10 +2,12 @@ # @LAYER: Infra # @BRIEF Chart domain mixin for SupersetClient — list, get, extract IDs from layout. # @RELATION DEPENDS_ON -> [SupersetClientBase] -import json import re -from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from typing import Any, cast + +from ..logger import belief_scope +from ..logger import logger as app_logger + app_logger = cast(Any, app_logger) # #region SupersetChartsMixin [C:3] [TYPE Class] # @BRIEF Mixin providing all chart-related Superset API operations. @@ -14,15 +16,15 @@ class SupersetChartsMixin: # #region SupersetClientGetChart [TYPE Function] [C:3] # @PURPOSE: Fetches a single chart by ID. # @RELATION: CALLS -> [APIClient] - def get_chart(self, chart_id: int) -> Dict: + def get_chart(self, chart_id: int) -> dict: with belief_scope("SupersetClient.get_chart", f"id={chart_id}"): response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}") - return cast(Dict, response) + return cast(dict, response) # #endregion SupersetClientGetChart # #region SupersetClientGetCharts [TYPE Function] [C:3] # @PURPOSE: Fetches all charts with pagination support. # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: + def get_charts(self, query: dict | None = None) -> tuple[int, list[dict]]: with belief_scope("get_charts"): validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: @@ -66,4 +68,4 @@ class SupersetChartsMixin: return found # #endregion SupersetClientExtractChartIdsFromLayout # #endregion SupersetChartsMixin -# #endregion SupersetChartsMixin \ No newline at end of file +# #endregion SupersetChartsMixin diff --git a/backend/src/core/superset_client/_dashboards_crud.py b/backend/src/core/superset_client/_dashboards_crud.py index d10f3775..e55b2007 100644 --- a/backend/src/core/superset_client/_dashboards_crud.py +++ b/backend/src/core/superset_client/_dashboards_crud.py @@ -7,9 +7,13 @@ import json import re from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, cast + from requests import Response -from ..logger import logger as app_logger, belief_scope + +from ..logger import belief_scope +from ..logger import logger as app_logger + app_logger = cast(Any, app_logger) # #region SupersetDashboardsCrudMixin [C:3] [TYPE Class] # @BRIEF Mixin providing dashboard detail resolution, export, import, and delete operations. @@ -21,18 +25,18 @@ class SupersetDashboardsCrudMixin: # @PURPOSE: Fetches detailed dashboard information including related charts and datasets. # @RELATION: CALLS -> [SupersetClientGetDashboard] # @RELATION: CALLS -> [SupersetClientGetChart] - def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict: + def get_dashboard_detail(self, dashboard_ref: int | str) -> dict: with belief_scope( "SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}" ): dashboard_response = self.get_dashboard(dashboard_ref) dashboard_data = dashboard_response.get("result", dashboard_response) - charts: List[Dict] = [] - datasets: List[Dict] = [] + charts: list[dict] = [] + datasets: list[dict] = [] # #region extract_dataset_id_from_form_data [TYPE Function] def extract_dataset_id_from_form_data( - form_data: Optional[Dict], - ) -> Optional[int]: + form_data: dict | None, + ) -> int | None: if not isinstance(form_data, dict): return None datasource = form_data.get("datasource") @@ -195,7 +199,7 @@ class SupersetDashboardsCrudMixin: known_dataset_ids = { d.get("id") for d in datasets if d.get("id") is not None } - missing_dataset_ids: List[int] = [] + missing_dataset_ids: list[int] = [] for raw_dataset_id in dataset_ids_from_charts: if raw_dataset_id is None or raw_dataset_id in known_dataset_ids: continue @@ -254,7 +258,7 @@ class SupersetDashboardsCrudMixin: # @PURPOSE: Экспортирует дашборд в виде ZIP-архива. # @SIDE_EFFECT: Performs network I/O to download archive. # @RELATION: CALLS -> [APIClient] - def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]: + def export_dashboard(self, dashboard_id: int) -> tuple[bytes, str]: with belief_scope("export_dashboard"): app_logger.info( "[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id @@ -282,10 +286,10 @@ class SupersetDashboardsCrudMixin: # @RELATION: CALLS -> [APIClient] def import_dashboard( self, - file_name: Union[str, Path], - dash_id: Optional[int] = None, - dash_slug: Optional[str] = None, - ) -> Dict: + file_name: str | Path, + dash_id: int | None = None, + dash_slug: str | None = None, + ) -> dict: with belief_scope("import_dashboard"): if file_name is None: raise ValueError("file_name cannot be None") @@ -317,7 +321,7 @@ class SupersetDashboardsCrudMixin: # @PURPOSE: Удаляет дашборд по его ID или slug. # @SIDE_EFFECT: Deletes resource from upstream Superset environment. # @RELATION: CALLS -> [APIClient] - def delete_dashboard(self, dashboard_id: Union[int, str]) -> None: + def delete_dashboard(self, dashboard_id: int | str) -> None: with belief_scope("delete_dashboard"): app_logger.info( "[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id @@ -325,7 +329,7 @@ class SupersetDashboardsCrudMixin: response = self.network.request( method="DELETE", endpoint=f"/dashboard/{dashboard_id}" ) - response = cast(Dict, response) + response = cast(dict, response) if response.get("result", True) is not False: app_logger.info( "[delete_dashboard][Success] Dashboard %s deleted.", dashboard_id @@ -337,4 +341,4 @@ class SupersetDashboardsCrudMixin: ) # #endregion SupersetClientDeleteDashboard # #endregion SupersetDashboardsCrudMixin -# #endregion SupersetDashboardsCrudMixin \ No newline at end of file +# #endregion SupersetDashboardsCrudMixin diff --git a/backend/src/core/superset_client/_dashboards_filters.py b/backend/src/core/superset_client/_dashboards_filters.py index 8cad8f82..14b5fdd4 100644 --- a/backend/src/core/superset_client/_dashboards_filters.py +++ b/backend/src/core/superset_client/_dashboards_filters.py @@ -3,8 +3,11 @@ # @BRIEF Dashboard native filter extraction mixin for SupersetClient. # @RELATION DEPENDS_ON -> [SupersetClientBase] import json -from typing import Any, Dict, Optional, Union, cast -from ..logger import logger as app_logger, belief_scope +from typing import Any, cast + +from ..logger import belief_scope +from ..logger import logger as app_logger + app_logger = cast(Any, app_logger) # #region SupersetDashboardsFiltersMixin [C:3] [TYPE Class] # @BRIEF Mixin providing dashboard native filter extraction from permalink and URL state. @@ -13,31 +16,31 @@ class SupersetDashboardsFiltersMixin: # #region SupersetClientGetDashboard [TYPE Function] [C:3] # @PURPOSE: Fetches a single dashboard by ID or slug. # @RELATION: CALLS -> [APIClient] - def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict: + def get_dashboard(self, dashboard_ref: int | str) -> dict: with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"): response = self.network.request( method="GET", endpoint=f"/dashboard/{dashboard_ref}" ) - return cast(Dict, response) + return cast(dict, response) # #endregion SupersetClientGetDashboard # #region SupersetClientGetDashboardPermalinkState [TYPE Function] [C:2] # @PURPOSE: Fetches stored dashboard permalink state by permalink key. # @RELATION: CALLS -> [APIClient] - def get_dashboard_permalink_state(self, permalink_key: str) -> Dict: + def get_dashboard_permalink_state(self, permalink_key: str) -> dict: with belief_scope( "SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}" ): response = self.network.request( method="GET", endpoint=f"/dashboard/permalink/{permalink_key}" ) - return cast(Dict, response) + return cast(dict, response) # #endregion SupersetClientGetDashboardPermalinkState # #region SupersetClientGetNativeFilterState [TYPE Function] [C:2] # @PURPOSE: Fetches stored native filter state by filter state key. # @RELATION: CALLS -> [APIClient] def get_native_filter_state( - self, dashboard_id: Union[int, str], filter_state_key: str - ) -> Dict: + self, dashboard_id: int | str, filter_state_key: str + ) -> dict: with belief_scope( "SupersetClient.get_native_filter_state", f"dashboard={dashboard_id}, key={filter_state_key}", @@ -46,12 +49,12 @@ class SupersetDashboardsFiltersMixin: method="GET", endpoint=f"/dashboard/{dashboard_id}/filter_state/{filter_state_key}", ) - return cast(Dict, response) + return cast(dict, response) # #endregion SupersetClientGetNativeFilterState # #region SupersetClientExtractNativeFiltersFromPermalink [TYPE Function] [C:3] # @PURPOSE: Extract native filters dataMask from a permalink key. # @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState] - def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict: + def extract_native_filters_from_permalink(self, permalink_key: str) -> dict: with belief_scope( "SupersetClient.extract_native_filters_from_permalink", f"key={permalink_key}", @@ -81,8 +84,8 @@ class SupersetDashboardsFiltersMixin: # @PURPOSE: Extract native filters from a native_filters_key URL parameter. # @RELATION: CALLS -> [SupersetClientGetNativeFilterState] def extract_native_filters_from_key( - self, dashboard_id: Union[int, str], filter_state_key: str - ) -> Dict: + self, dashboard_id: int | str, filter_state_key: str + ) -> dict: with belief_scope( "SupersetClient.extract_native_filters_from_key", f"dashboard={dashboard_id}, key={filter_state_key}", @@ -132,7 +135,7 @@ class SupersetDashboardsFiltersMixin: # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present. # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink] # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey] - def parse_dashboard_url_for_filters(self, url: str) -> Dict: + def parse_dashboard_url_for_filters(self, url: str) -> dict: with belief_scope( "SupersetClient.parse_dashboard_url_for_filters", f"url={url}" ): @@ -222,4 +225,4 @@ class SupersetDashboardsFiltersMixin: return result # #endregion SupersetClientParseDashboardUrlForFilters # #endregion SupersetDashboardsFiltersMixin -# #endregion SupersetDashboardsFiltersMixin \ No newline at end of file +# #endregion SupersetDashboardsFiltersMixin diff --git a/backend/src/core/superset_client/_dashboards_list.py b/backend/src/core/superset_client/_dashboards_list.py index 18e78554..f4720fd8 100644 --- a/backend/src/core/superset_client/_dashboards_list.py +++ b/backend/src/core/superset_client/_dashboards_list.py @@ -4,8 +4,11 @@ # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetUserProjectionMixin] import json -from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from typing import Any, cast + +from ..logger import belief_scope +from ..logger import logger as app_logger + app_logger = cast(Any, app_logger) # #region SupersetDashboardsListMixin [C:3] [TYPE Class] # @BRIEF Mixin providing dashboard listing and summary projection operations. @@ -15,7 +18,7 @@ class SupersetDashboardsListMixin: # #region SupersetClientGetDashboards [TYPE Function] [C:3] # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию. # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: + def get_dashboards(self, query: dict | None = None) -> tuple[int, list[dict]]: with belief_scope("get_dashboards"): app_logger.info("[get_dashboards][Enter] Fetching dashboards.") validated_query = self._validate_query_params(query or {}) @@ -39,8 +42,8 @@ class SupersetDashboardsListMixin: # @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages. # @RELATION: CALLS -> [APIClient] def get_dashboards_page( - self, query: Optional[Dict] = None - ) -> Tuple[int, List[Dict]]: + self, query: dict | None = None + ) -> tuple[int, list[dict]]: with belief_scope("get_dashboards_page"): validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: @@ -49,7 +52,7 @@ class SupersetDashboardsListMixin: "published", "created_by", "changed_by", "changed_by_name", "owners", ] response_json = cast( - Dict[str, Any], + dict[str, Any], self.network.request( method="GET", endpoint="/dashboard/", @@ -63,9 +66,9 @@ class SupersetDashboardsListMixin: # #region SupersetClientGetDashboardsSummary [TYPE Function] [C:3] # @PURPOSE: Fetches dashboard metadata optimized for the grid. # @RELATION: CALLS -> [SupersetClientGetDashboards] - def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]: + def get_dashboards_summary(self, require_slug: bool = False) -> list[dict]: with belief_scope("SupersetClient.get_dashboards_summary"): - query: Dict[str, Any] = {} + query: dict[str, Any] = {} if require_slug: query["filters"] = [{"col": "slug", "opr": "neq", "value": ""}] _, dashboards = self.get_dashboards(query=query) @@ -83,7 +86,7 @@ class SupersetDashboardsListMixin: projected_modified_by = self._extract_user_display( raw_changed_by_name, raw_changed_by, ) - raw_owner_usernames: List[str] = [] + raw_owner_usernames: list[str] = [] if isinstance(raw_owners, list): for owner_payload in raw_owners: if isinstance(owner_payload, dict): @@ -121,15 +124,15 @@ class SupersetDashboardsListMixin: self, page: int, page_size: int, - search: Optional[str] = None, + search: str | None = None, require_slug: bool = False, - ) -> Tuple[int, List[Dict]]: + ) -> tuple[int, list[dict]]: with belief_scope("SupersetClient.get_dashboards_summary_page"): - query: Dict[str, Any] = { + query: dict[str, Any] = { "page": max(page - 1, 0), "page_size": page_size, } - filters: List[Dict[str, Any]] = [] + filters: list[dict[str, Any]] = [] if require_slug: filters.append({"col": "slug", "opr": "neq", "value": ""}) normalized_search = (search or "").strip() @@ -165,4 +168,4 @@ class SupersetDashboardsListMixin: return total_count, result # #endregion SupersetClientGetDashboardsSummaryPage # #endregion SupersetDashboardsListMixin -# #endregion SupersetDashboardsListMixin \ No newline at end of file +# #endregion SupersetDashboardsListMixin diff --git a/backend/src/core/superset_client/_databases.py b/backend/src/core/superset_client/_databases.py index 80465877..0c837e4f 100644 --- a/backend/src/core/superset_client/_databases.py +++ b/backend/src/core/superset_client/_databases.py @@ -2,8 +2,11 @@ # @LAYER: Infra # @BRIEF Database domain mixin for SupersetClient — list, get, summary, by_uuid. # @RELATION DEPENDS_ON -> [SupersetClientBase] -from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from typing import Any, cast + +from ..logger import belief_scope +from ..logger import logger as app_logger + app_logger = cast(Any, app_logger) # #region SupersetDatabasesMixin [C:3] [TYPE Class] # @BRIEF Mixin providing all database-related Superset API operations. @@ -12,7 +15,7 @@ class SupersetDatabasesMixin: # #region SupersetClientGetDatabases [TYPE Function] [C:3] # @PURPOSE: Получает полный список баз данных. # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: + def get_databases(self, query: dict | None = None) -> tuple[int, list[dict]]: with belief_scope("get_databases"): app_logger.info("[get_databases][Enter] Fetching databases.") validated_query = self._validate_query_params(query or {}) @@ -32,20 +35,20 @@ class SupersetDatabasesMixin: # #region SupersetClientGetDatabase [TYPE Function] [C:3] # @PURPOSE: Получает информацию о конкретной базе данных по её ID. # @RELATION: CALLS -> [APIClient] - def get_database(self, database_id: int) -> Dict: + def get_database(self, database_id: int) -> dict: with belief_scope("get_database"): app_logger.info("[get_database][Enter] Fetching database %s.", database_id) response = self.network.request( method="GET", endpoint=f"/database/{database_id}" ) - response = cast(Dict, response) + response = cast(dict, response) app_logger.reflect(f"Got database {database_id}.", extra={"src": "get_database"}) return response # #endregion SupersetClientGetDatabase # #region SupersetClientGetDatabasesSummary [TYPE Function] [C:3] # @PURPOSE: Fetch a summary of databases including uuid, name, and engine. # @RELATION: CALLS -> [SupersetClientGetDatabases] - def get_databases_summary(self) -> List[Dict]: + def get_databases_summary(self) -> list[dict]: with belief_scope("SupersetClient.get_databases_summary"): query = {"columns": ["id", "uuid", "database_name", "backend"]} _, databases = self.get_databases(query=query) @@ -57,11 +60,11 @@ class SupersetDatabasesMixin: # #region SupersetClientGetDatabaseByUuid [TYPE Function] [C:3] # @PURPOSE: Find a database by its UUID. # @RELATION: CALLS -> [SupersetClientGetDatabases] - def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]: + def get_database_by_uuid(self, db_uuid: str) -> dict | None: with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"): query = {"filters": [{"col": "uuid", "op": "eq", "value": db_uuid}]} _, databases = self.get_databases(query=query) return databases[0] if databases else None # #endregion SupersetClientGetDatabaseByUuid # #endregion SupersetDatabasesMixin -# #endregion SupersetDatabasesMixin \ No newline at end of file +# #endregion SupersetDatabasesMixin diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index eeca4672..6e780f48 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -3,8 +3,11 @@ # @BRIEF Dataset domain mixin for SupersetClient — list, get, detail, update. # @RELATION DEPENDS_ON -> [SupersetClientBase] import json -from typing import Any, Dict, List, Optional, Tuple, cast -from ..logger import logger as app_logger, belief_scope +from typing import Any, cast + +from ..logger import belief_scope +from ..logger import logger as app_logger + app_logger = cast(Any, app_logger) # #region SupersetDatasetsMixin [C:3] [TYPE Class] # @BRIEF Mixin providing basic dataset CRUD operations (list, get, detail, update). @@ -13,7 +16,7 @@ class SupersetDatasetsMixin: # #region SupersetClientGetDatasets [TYPE Function] [C:3] # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию. # @RELATION: CALLS -> [SupersetClientFetchAllPages] - def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: + def get_datasets(self, query: dict | None = None) -> tuple[int, list[dict]]: with belief_scope("get_datasets"): app_logger.info("[get_datasets][Enter] Fetching datasets.") validated_query = self._validate_query_params(query) @@ -31,7 +34,7 @@ class SupersetDatasetsMixin: # #region SupersetClientGetDatasetsSummary [TYPE Function] [C:3] # @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid. # @RELATION: CALLS -> [SupersetClientGetDatasets] - def get_datasets_summary(self) -> List[Dict]: + def get_datasets_summary(self) -> list[dict]: with belief_scope("SupersetClient.get_datasets_summary"): query = {"columns": ["id", "table_name", "schema", "database"]} _, datasets = self.get_datasets(query=query) @@ -53,7 +56,7 @@ class SupersetDatasetsMixin: # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards. # @RELATION: CALLS -> [SupersetClientGetDataset] # @RELATION: CALLS -> [APIClient] - def get_dataset_detail(self, dataset_id: int) -> Dict: + def get_dataset_detail(self, dataset_id: int) -> dict: with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"): def as_bool(value, default=False): if value is None: @@ -153,13 +156,13 @@ class SupersetDatasetsMixin: # #region SupersetClientGetDataset [TYPE Function] [C:3] # @PURPOSE: Получает информацию о конкретном датасете по его ID. # @RELATION: CALLS -> [APIClient] - def get_dataset(self, dataset_id: int) -> Dict: + def get_dataset(self, dataset_id: int) -> dict: with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"): app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id) response = self.network.request( method="GET", endpoint=f"/dataset/{dataset_id}" ) - response = cast(Dict, response) + response = cast(dict, response) app_logger.reflect(f"Got dataset {dataset_id}.", extra={"src": "get_dataset"}) return response # #endregion SupersetClientGetDataset @@ -167,7 +170,7 @@ class SupersetDatasetsMixin: # @PURPOSE: Обновляет данные датасета по его ID. # @SIDE_EFFECT: Modifies resource in upstream Superset environment. # @RELATION: CALLS -> [APIClient] - def update_dataset(self, dataset_id: int, data: Dict) -> Dict: + def update_dataset(self, dataset_id: int, data: dict) -> dict: with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"): app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id) response = self.network.request( @@ -176,9 +179,9 @@ class SupersetDatasetsMixin: data=json.dumps(data), headers={"Content-Type": "application/json"}, ) - response = cast(Dict, response) + response = cast(dict, response) app_logger.reflect(f"Updated dataset {dataset_id}.", extra={"src": "update_dataset"}) return response # #endregion SupersetClientUpdateDataset # #endregion SupersetDatasetsMixin -# #endregion SupersetDatasetsMixin \ No newline at end of file +# #endregion SupersetDatasetsMixin diff --git a/backend/src/core/superset_client/_datasets_preview.py b/backend/src/core/superset_client/_datasets_preview.py index 93383849..02d0c8b7 100644 --- a/backend/src/core/superset_client/_datasets_preview.py +++ b/backend/src/core/superset_client/_datasets_preview.py @@ -5,15 +5,19 @@ # @RELATION DEPENDS_ON -> [SupersetDatasetsMixin] import json from copy import deepcopy -from typing import Any, Dict, List, Optional -from ..logger import logger as app_logger, belief_scope +from typing import Any + +from ..logger import belief_scope +from ..logger import logger as app_logger from ..utils.network import SupersetAPIError + + # #region SupersetDatasetsPreviewMixin [C:4] [TYPE Class] # @BRIEF Mixin providing dataset preview compilation and query context building. class SupersetDatasetsPreviewMixin: # #region SupersetClientCompileDatasetPreview [TYPE Function] [C:4] # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output. - def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: + def compile_dataset_preview(self, dataset_id: int, template_params: dict[str, Any] | None = None, effective_filters: list[dict[str, Any]] | None = None) -> dict[str, Any]: with belief_scope('SupersetClientCompileDatasetPreview'): app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview') dataset_response = self.get_dataset(dataset_id) @@ -22,8 +26,8 @@ class SupersetDatasetsPreviewMixin: legacy_form_data = self.build_dataset_preview_legacy_form_data(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or []) legacy_form_data_payload = json.dumps(legacy_form_data, sort_keys=True, default=str) request_payload = json.dumps(query_context) - strategy_attempts: List[Dict[str, Any]] = [] - strategy_candidates: List[Dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}] + strategy_attempts: list[dict[str, Any]] = [] + strategy_candidates: list[dict[str, Any]] = [{'endpoint_kind': 'legacy_explore_form_data', 'endpoint': '/explore_json/form_data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'legacy_data_form_data', 'endpoint': '/data', 'request_transport': 'query_param_form_data', 'params': {'form_data': legacy_form_data_payload}}, {'endpoint_kind': 'v1_chart_data', 'endpoint': '/chart/data', 'request_transport': 'json_body', 'data': request_payload, 'headers': {'Content-Type': 'application/json'}}] for candidate in strategy_candidates: endpoint_kind = candidate['endpoint_kind'] endpoint_path = candidate['endpoint'] @@ -32,7 +36,7 @@ class SupersetDatasetsPreviewMixin: request_body = candidate.get('data') request_headers = deepcopy(candidate.get('headers') or {}) request_param_keys = sorted(request_params.keys()) - request_payload_keys: List[str] = [] + request_payload_keys: list[str] = [] if isinstance(request_body, str): try: decoded_request_body = json.loads(request_body) @@ -64,7 +68,7 @@ class SupersetDatasetsPreviewMixin: # #endregion SupersetClientCompileDatasetPreview # #region SupersetClientBuildDatasetPreviewLegacyFormData [TYPE Function] [C:4] # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic. - def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]: + def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: dict[str, Any], template_params: dict[str, Any], effective_filters: list[dict[str, Any]]) -> dict[str, Any]: with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'): app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters) @@ -97,10 +101,10 @@ class SupersetDatasetsPreviewMixin: def build_dataset_preview_query_context( self, dataset_id: int, - dataset_record: Dict[str, Any], - template_params: Dict[str, Any], - effective_filters: List[Dict[str, Any]], - ) -> Dict[str, Any]: + dataset_record: dict[str, Any], + template_params: dict[str, Any], + effective_filters: list[dict[str, Any]], + ) -> dict[str, Any]: with belief_scope("SupersetClientBuildDatasetPreviewQueryContext"): app_logger.reason( "Building Superset dataset preview query context", @@ -114,7 +118,7 @@ class SupersetDatasetsPreviewMixin: ) normalized_filters = normalized_filter_payload["filters"] normalized_extra_form_data = normalized_filter_payload["extra_form_data"] - datasource_payload: Dict[str, Any] = { + datasource_payload: dict[str, Any] = { "id": dataset_id, "type": "table", } @@ -143,10 +147,10 @@ class SupersetDatasetsPreviewMixin: "Dataset template_params could not be parsed while building preview query context", extra={"dataset_id": dataset_id}, ) - extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data) + extra_form_data: dict[str, Any] = deepcopy(normalized_extra_form_data) if normalized_filters: extra_form_data["filters"] = deepcopy(normalized_filters) - query_object: Dict[str, Any] = { + query_object: dict[str, Any] = { "filters": normalized_filters, "extras": {"where": ""}, "columns": [], @@ -170,7 +174,7 @@ class SupersetDatasetsPreviewMixin: extra_form_data["time_range"] = time_range result_format = dataset_record.get("result_format") or "json" result_type = "query" - form_data: Dict[str, Any] = { + form_data: dict[str, Any] = { "datasource": f"{datasource_payload['id']}__{datasource_payload['type']}", "datasource_id": datasource_payload["id"], "datasource_type": datasource_payload["type"], @@ -207,4 +211,4 @@ class SupersetDatasetsPreviewMixin: return payload # #endregion SupersetClientBuildDatasetPreviewQueryContext # #endregion SupersetDatasetsPreviewMixin -# #endregion SupersetDatasetsPreviewMixin \ No newline at end of file +# #endregion SupersetDatasetsPreviewMixin diff --git a/backend/src/core/superset_client/_datasets_preview_filters.py b/backend/src/core/superset_client/_datasets_preview_filters.py index f2277776..d9623c2c 100644 --- a/backend/src/core/superset_client/_datasets_preview_filters.py +++ b/backend/src/core/superset_client/_datasets_preview_filters.py @@ -4,9 +4,13 @@ # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewMixin] from copy import deepcopy -from typing import Any, Dict, List, Optional, Tuple -from ..logger import logger as app_logger, belief_scope +from typing import Any + +from ..logger import belief_scope +from ..logger import logger as app_logger from ..utils.network import SupersetAPIError + + # #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Class] # @BRIEF Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] @@ -16,14 +20,14 @@ class SupersetDatasetsPreviewFiltersMixin: # @PURPOSE: Convert execution mappings into Superset chart-data filter objects. def _normalize_effective_filters_for_query_context( self, - effective_filters: List[Dict[str, Any]], - ) -> Dict[str, Any]: + effective_filters: list[dict[str, Any]], + ) -> dict[str, Any]: with belief_scope( "SupersetClient._normalize_effective_filters_for_query_context" ): - normalized_filters: List[Dict[str, Any]] = [] - merged_extra_form_data: Dict[str, Any] = {} - diagnostics: List[Dict[str, Any]] = [] + normalized_filters: list[dict[str, Any]] = [] + merged_extra_form_data: dict[str, Any] = {} + diagnostics: list[dict[str, Any]] = [] for item in effective_filters: if not isinstance(item, dict): continue @@ -35,8 +39,8 @@ class SupersetDatasetsPreviewFiltersMixin: ).strip() value = item.get("effective_value") normalized_payload = item.get("normalized_filter_payload") - preserved_clauses: List[Dict[str, Any]] = [] - preserved_extra_form_data: Dict[str, Any] = {} + preserved_clauses: list[dict[str, Any]] = [] + preserved_extra_form_data: dict[str, Any] = {} used_preserved_clauses = False if isinstance(normalized_payload, dict): raw_clauses = normalized_payload.get("filter_clauses") @@ -54,7 +58,7 @@ class SupersetDatasetsPreviewFiltersMixin: if key == "filters": continue merged_extra_form_data[key] = deepcopy(extra_value) - outgoing_clauses: List[Dict[str, Any]] = [] + outgoing_clauses: list[dict[str, Any]] = [] if preserved_clauses: for clause in preserved_clauses: clause_copy = deepcopy(clause) @@ -109,13 +113,13 @@ class SupersetDatasetsPreviewFiltersMixin: # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses. def _extract_compiled_sql_from_preview_response( self, response: Any - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("SupersetClient._extract_compiled_sql_from_preview_response"): if not isinstance(response, dict): raise SupersetAPIError( "Superset preview response was not a JSON object" ) - response_diagnostics: List[Dict[str, Any]] = [] + response_diagnostics: list[dict[str, Any]] = [] result_payload = response.get("result") if isinstance(result_payload, list): for index, item in enumerate(result_payload): @@ -143,7 +147,7 @@ class SupersetDatasetsPreviewFiltersMixin: "raw_response": response, "response_diagnostics": response_diagnostics, } - top_level_candidates: List[Tuple[str, Any]] = [ + top_level_candidates: list[tuple[str, Any]] = [ ("query", response.get("query")), ("sql", response.get("sql")), ("compiled_sql", response.get("compiled_sql")), @@ -173,4 +177,4 @@ class SupersetDatasetsPreviewFiltersMixin: ) # #endregion SupersetClientExtractCompiledSqlFromPreviewResponse # #endregion SupersetDatasetsPreviewFiltersMixin -# #endregion SupersetDatasetsPreviewFiltersMixin \ No newline at end of file +# #endregion SupersetDatasetsPreviewFiltersMixin diff --git a/backend/src/core/superset_client/_user_projection.py b/backend/src/core/superset_client/_user_projection.py index 03af124a..d2fdd12c 100644 --- a/backend/src/core/superset_client/_user_projection.py +++ b/backend/src/core/superset_client/_user_projection.py @@ -2,24 +2,26 @@ # @LAYER: Infra # @BRIEF User/owner payload normalization helpers for Superset client responses. # @RELATION DEPENDS_ON -> [SupersetClientBase] -from typing import Any, Dict, List, Optional, Union +from typing import Any + + # #region SupersetUserProjectionMixin [C:2] [TYPE Class] # @BRIEF Mixin providing user/owner payload normalization for Superset API responses. # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetUserProjectionMixin: # #region SupersetClientExtractOwnerLabels [TYPE Function] [C:1] # @PURPOSE: Normalize dashboard owners payload to stable display labels. - def _extract_owner_labels(self, owners_payload: Any) -> List[str]: + def _extract_owner_labels(self, owners_payload: Any) -> list[str]: if owners_payload is None: return [] - owners_list: List[Any] + owners_list: list[Any] if isinstance(owners_payload, list): owners_list = owners_payload else: owners_list = [owners_payload] - normalized: List[str] = [] + normalized: list[str] = [] for owner in owners_list: - label: Optional[str] = None + label: str | None = None if isinstance(owner, dict): label = self._extract_user_display(None, owner) else: @@ -31,8 +33,8 @@ class SupersetUserProjectionMixin: # #region SupersetClientExtractUserDisplay [TYPE Function] [C:1] # @PURPOSE: Normalize user payload to a stable display name. def _extract_user_display( - self, preferred_value: Optional[str], user_payload: Optional[Dict] - ) -> Optional[str]: + self, preferred_value: str | None, user_payload: dict | None + ) -> str | None: preferred = self._sanitize_user_text(preferred_value) if preferred: return preferred @@ -57,7 +59,7 @@ class SupersetUserProjectionMixin: # #endregion SupersetClientExtractUserDisplay # #region SupersetClientSanitizeUserText [TYPE Function] [C:1] # @PURPOSE: Convert scalar value to non-empty user-facing text. - def _sanitize_user_text(self, value: Optional[Union[str, int]]) -> Optional[str]: + def _sanitize_user_text(self, value: str | int | None) -> str | None: if value is None: return None normalized = str(value).strip() @@ -66,4 +68,4 @@ class SupersetUserProjectionMixin: return normalized # #endregion SupersetClientSanitizeUserText # #endregion SupersetUserProjectionMixin -# #endregion SupersetUserProjection \ No newline at end of file +# #endregion SupersetUserProjection diff --git a/backend/src/core/superset_profile_lookup.py b/backend/src/core/superset_profile_lookup.py index bba5933b..7b63405f 100644 --- a/backend/src/core/superset_profile_lookup.py +++ b/backend/src/core/superset_profile_lookup.py @@ -8,9 +8,12 @@ # # @INVARIANT: Adapter never leaks raw upstream payload shape to API consumers. import json -from typing import Any, Dict, List, Optional -from .logger import logger, belief_scope +from typing import Any + +from .logger import belief_scope, logger from .utils.network import APIClient, AuthenticationError, SupersetAPIError + + # #region SupersetAccountLookupAdapter [C:3] [TYPE Class] # @BRIEF Lookup Superset users and normalize candidates for profile binding. # @RELATION DEPENDS_ON -> [APIClient] @@ -31,12 +34,12 @@ class SupersetAccountLookupAdapter: # @RETURN: Dict[str, Any] def get_users_page( self, - search: Optional[str] = None, + search: str | None = None, page_index: int = 0, page_size: int = 20, sort_column: str = "username", sort_order: str = "desc", - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("SupersetAccountLookupAdapter.get_users_page"): normalized_page_index = max(int(page_index), 0) normalized_page_size = max(int(page_size), 1) @@ -44,7 +47,7 @@ class SupersetAccountLookupAdapter: normalized_sort_order = str(sort_order or "desc").strip().lower() if normalized_sort_order not in {"asc", "desc"}: normalized_sort_order = "desc" - query: Dict[str, Any] = { + query: dict[str, Any] = { "page": normalized_page_index, "page_size": normalized_page_size, "order_column": normalized_sort_column, @@ -61,8 +64,8 @@ class SupersetAccountLookupAdapter: "Prepared Superset users lookup query", extra={"src": "SupersetAccountLookupAdapter.get_users_page", "payload": {"env": self.environment_id, "order_column": normalized_sort_column, "sort_order": normalized_sort_order, "direction": query.get("order_direction")}}, ) - primary_error: Optional[Exception] = None - last_error: Optional[Exception] = None + primary_error: Exception | None = None + last_error: Exception | None = None for attempt_index, endpoint in enumerate(("/security/users/", "/security/users"), start=1): try: logger.reason( @@ -124,12 +127,12 @@ class SupersetAccountLookupAdapter: response: Any, page_index: int, page_size: int, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("SupersetAccountLookupAdapter._normalize_lookup_payload"): payload = response if isinstance(payload, dict) and isinstance(payload.get("result"), dict): payload = payload.get("result") - raw_items: List[Any] = [] + raw_items: list[Any] = [] total = 0 if isinstance(payload, dict): if isinstance(payload.get("result"), list): @@ -144,7 +147,7 @@ class SupersetAccountLookupAdapter: elif isinstance(payload, list): raw_items = payload total = len(raw_items) - normalized_items: List[Dict[str, Any]] = [] + normalized_items: list[dict[str, Any]] = [] seen_usernames = set() for raw_user in raw_items: candidate = self.normalize_user_payload(raw_user) @@ -173,7 +176,7 @@ class SupersetAccountLookupAdapter: # @PRE: raw_user may have heterogenous key names between Superset versions. # @POST: Returns normalized candidate keys (environment_id, username, display_name, email, is_active). # @RETURN: Dict[str, Any] - def normalize_user_payload(self, raw_user: Any) -> Dict[str, Any]: + def normalize_user_payload(self, raw_user: Any) -> dict[str, Any]: if not isinstance(raw_user, dict): raw_user = {} username = str( @@ -202,4 +205,4 @@ class SupersetAccountLookupAdapter: } # #endregion normalize_user_payload # #endregion SupersetAccountLookupAdapter -# #endregion SupersetProfileLookup \ No newline at end of file +# #endregion SupersetProfileLookup diff --git a/backend/src/core/task_manager/__init__.py b/backend/src/core/task_manager/__init__.py index aa27376a..997ddd9a 100644 --- a/backend/src/core/task_manager/__init__.py +++ b/backend/src/core/task_manager/__init__.py @@ -1,8 +1,8 @@ # #region TaskManagerPackage [TYPE Module] [SEMANTICS task, manager, package, init] -from .models import Task, TaskStatus, LogEntry from .manager import TaskManager +from .models import LogEntry, Task, TaskStatus -__all__ = ["TaskManager", "Task", "TaskStatus", "LogEntry"] +__all__ = ["LogEntry", "Task", "TaskManager", "TaskStatus"] # #endregion TaskManagerPackage diff --git a/backend/src/core/task_manager/__tests__/test_context.py b/backend/src/core/task_manager/__tests__/test_context.py index 3ebe4dba..93d3a0c7 100644 --- a/backend/src/core/task_manager/__tests__/test_context.py +++ b/backend/src/core/task_manager/__tests__/test_context.py @@ -2,7 +2,10 @@ # @RELATION BELONGS_TO -> SrcRoot # @BRIEF Verify TaskContext preserves optional background task scheduler across sub-context creation. from unittest.mock import MagicMock + from src.core.task_manager.context import TaskContext + + # #region test_task_context_preserves_background_tasks_across_sub_context [TYPE Function] # @RELATION BINDS_TO -> TestContext # @BRIEF Plugins must be able to access background_tasks from both root and sub-context loggers. @@ -20,4 +23,4 @@ def test_task_context_preserves_background_tasks_across_sub_context(): assert context.background_tasks is background_tasks assert sub_context.background_tasks is background_tasks # #endregion test_task_context_preserves_background_tasks_across_sub_context -# #endregion TestContext \ No newline at end of file +# #endregion TestContext diff --git a/backend/src/core/task_manager/__tests__/test_task_logger.py b/backend/src/core/task_manager/__tests__/test_task_logger.py index 7acb6547..48d623c4 100644 --- a/backend/src/core/task_manager/__tests__/test_task_logger.py +++ b/backend/src/core/task_manager/__tests__/test_task_logger.py @@ -2,9 +2,13 @@ # @RELATION VERIFIES -> ../task_logger.py # @BRIEF Contract testing for TaskLogger # #endregion __tests__/test_task_logger -import pytest from unittest.mock import MagicMock + +import pytest + from src.core.task_manager.task_logger import TaskLogger + + # @TEST_FIXTURE: valid_task_logger -> {"task_id": "test_123", "add_log_fn": lambda *args: None, "source": "test_plugin"} @pytest.fixture def mock_add_log(): @@ -35,7 +39,7 @@ def test_log_methods_delegation(task_logger, mock_add_log): source="test_plugin", metadata={"k": "v"} ) - + task_logger.error("error message", source="override") mock_add_log.assert_called_with( task_id="test_123", @@ -105,11 +109,11 @@ def test_progress_log(task_logger, mock_add_log): source="test_plugin", metadata={"progress": 45.5} ) - + # Boundary checks task_logger.progress("Step high", 150) assert mock_add_log.call_args[1]["metadata"]["progress"] == 100 - + task_logger.progress("Step low", -10) assert mock_add_log.call_args[1]["metadata"]["progress"] == 0 -# #endregion test_progress_log \ No newline at end of file +# #endregion test_progress_log diff --git a/backend/src/core/task_manager/cleanup.py b/backend/src/core/task_manager/cleanup.py index 5ebd5b17..297b81a0 100644 --- a/backend/src/core/task_manager/cleanup.py +++ b/backend/src/core/task_manager/cleanup.py @@ -4,10 +4,12 @@ # @RELATION DEPENDS_ON -> [TaskPersistenceService] # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] # @RELATION DEPENDS_ON -> [ConfigManager] -from typing import List -from .persistence import TaskPersistenceService, TaskLogPersistenceService -from ..logger import logger, belief_scope + from ..config_manager import ConfigManager +from ..logger import belief_scope, logger +from .persistence import TaskLogPersistenceService, TaskPersistenceService + + # #region TaskCleanupService [C:3] [TYPE Class] # @BRIEF Provides methods to clean up old task records and their associated logs. # @RELATION DEPENDS_ON -> [TaskPersistenceService] @@ -36,23 +38,23 @@ class TaskCleanupService: with belief_scope("TaskCleanupService.run_cleanup"): settings = self.config_manager.get_config().settings retention_days = settings.task_retention_days - + logger.info(f"Cleaning up tasks older than {retention_days} days.") - + # Load tasks to check for limit tasks = self.persistence_service.load_tasks(limit=1000) if len(tasks) > settings.task_retention_limit: - to_delete: List[str] = [t.id for t in tasks[settings.task_retention_limit:]] - + to_delete: list[str] = [t.id for t in tasks[settings.task_retention_limit:]] + # Delete logs first (before task records) self.log_persistence_service.delete_logs_for_tasks(to_delete) - + # Then delete task records self.persistence_service.delete_tasks(to_delete) - + logger.info(f"Deleted {len(to_delete)} tasks and their logs exceeding limit of {settings.task_retention_limit}") # #endregion run_cleanup - + # #region delete_task_with_logs [TYPE Function] # @PURPOSE: Delete a single task and all its associated logs. # @PRE: task_id is a valid task ID. @@ -63,11 +65,11 @@ class TaskCleanupService: with belief_scope("TaskCleanupService.delete_task_with_logs", f"task_id={task_id}"): # Delete logs first self.log_persistence_service.delete_logs_for_task(task_id) - + # Then delete task record self.persistence_service.delete_tasks([task_id]) - + logger.info(f"Deleted task {task_id} and its associated logs") # #endregion delete_task_with_logs # #endregion TaskCleanupService -# #endregion TaskCleanupModule \ No newline at end of file +# #endregion TaskCleanupModule diff --git a/backend/src/core/task_manager/context.py b/backend/src/core/task_manager/context.py index 6b7521ee..64378374 100644 --- a/backend/src/core/task_manager/context.py +++ b/backend/src/core/task_manager/context.py @@ -8,9 +8,13 @@ # @POST: Plugins receive context instances with stable logger and parameter accessors. # @SIDE_EFFECT: Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts. # @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] -from typing import Dict, Any, Callable, Optional -from .task_logger import TaskLogger +from collections.abc import Callable +from typing import Any + from ..logger import belief_scope +from .task_logger import TaskLogger + + # #region TaskContext [C:5] [TYPE Class] [SEMANTICS context, task, execution, plugin] # @BRIEF A container passed to plugin.execute() providing the logger and other task-specific utilities. # @INVARIANT: logger is always a valid TaskLogger instance. @@ -56,9 +60,9 @@ class TaskContext: self, task_id: str, add_log_fn: Callable, - params: Dict[str, Any], + params: dict[str, Any], default_source: str = "plugin", - background_tasks: Optional[Any] = None, + background_tasks: Any | None = None, ): with belief_scope("__init__"): self._task_id = task_id @@ -94,7 +98,7 @@ class TaskContext: # @POST: Returns the parameters dictionary. # @RETURN: Dict[str, Any] - The task parameters. @property - def params(self) -> Dict[str, Any]: + def params(self) -> dict[str, Any]: with belief_scope("params"): return self._params # #endregion params @@ -103,7 +107,7 @@ class TaskContext: # @PRE: TaskContext must be initialized. # @POST: Returns BackgroundTasks-like object or None. @property - def background_tasks(self) -> Optional[Any]: + def background_tasks(self) -> Any | None: with belief_scope("background_tasks"): return self._background_tasks # #endregion background_tasks @@ -136,4 +140,4 @@ class TaskContext: ) # #endregion create_sub_context # #endregion TaskContext -# #endregion TaskContextModule \ No newline at end of file +# #endregion TaskContextModule diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index 7c30a5ea..d8724621 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -25,16 +25,19 @@ # @TEST_EDGE: external_failure -> {"db_unavailable": true} # @TEST_INVARIANT: logger_compliance -> verifies: [valid_module] import asyncio -import threading import inspect +import threading from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timezone -from typing import Dict, Any, List, Optional -from .models import Task, TaskStatus, LogEntry, LogFilter, LogStats -from .persistence import TaskPersistenceService, TaskLogPersistenceService -from .context import TaskContext -from ..logger import logger, belief_scope, should_log_task_level +from datetime import UTC, datetime +from typing import Any + from ..cot_logger import seed_trace_id +from ..logger import belief_scope, logger, should_log_task_level +from .context import TaskContext +from .models import LogEntry, LogFilter, LogStats, Task, TaskStatus +from .persistence import TaskLogPersistenceService, TaskPersistenceService + + # #region TaskManager [C:5] [TYPE Class] [SEMANTICS task, manager, lifecycle, execution, state] # @BRIEF Manages the lifecycle of tasks, including their creation, execution, and state tracking. # @LAYER: Core @@ -103,15 +106,15 @@ class TaskManager: with belief_scope("TaskManager.__init__"): logger.reason("Initializing task manager runtime services") self.plugin_loader = plugin_loader - self.tasks: Dict[str, Task] = {} - self.subscribers: Dict[str, List[asyncio.Queue]] = {} + self.tasks: dict[str, Task] = {} + self.subscribers: dict[str, list[asyncio.Queue]] = {} self.executor = ThreadPoolExecutor( max_workers=5 ) # For CPU-bound plugin execution self.persistence_service = TaskPersistenceService() self.log_persistence_service = TaskLogPersistenceService() # Log buffer: task_id -> List[LogEntry] - self._log_buffer: Dict[str, List[LogEntry]] = {} + self._log_buffer: dict[str, list[LogEntry]] = {} self._log_buffer_lock = threading.Lock() # Flusher thread for batch writing logs self._flusher_stop_event = threading.Event() @@ -123,7 +126,7 @@ class TaskManager: self.loop = asyncio.get_running_loop() except RuntimeError: self.loop = asyncio.get_event_loop() - self.task_futures: Dict[str, asyncio.Future] = {} + self.task_futures: dict[str, asyncio.Future] = {} # Load persisted tasks on startup self.load_persisted_tasks() logger.reflect( @@ -201,7 +204,7 @@ class TaskManager: # @RELATION: [DEPENDS_ON] ->[JobLifecycle] # @RELATION: [DEPENDS_ON] ->[TaskGraph] async def create_task( - self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None + self, plugin_id: str, params: dict[str, Any], user_id: str | None = None ) -> Task: with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"): if not self.plugin_loader.has_plugin(plugin_id): @@ -325,7 +328,7 @@ class TaskManager: # @THROWS: ValueError if task not found or not awaiting mapping. # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] # @RELATION: [DEPENDS_ON] ->[JobLifecycle] - async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]): + async def resolve_task(self, task_id: str, resolution_params: dict[str, Any]): with belief_scope("TaskManager.resolve_task", f"task_id={task_id}"): task = self.tasks.get(task_id) if not task or task.status != TaskStatus.AWAITING_MAPPING: @@ -386,7 +389,7 @@ class TaskManager: # @PARAM: task_id (str) - ID of the task. # @RETURN: Optional[Task] - The task or None. # @RELATION: [DEPENDS_ON] ->[TaskGraph] - def get_task(self, task_id: str) -> Optional[Task]: + def get_task(self, task_id: str) -> Task | None: with belief_scope("TaskManager.get_task", f"task_id={task_id}"): return self.tasks.get(task_id) # #endregion get_task @@ -396,7 +399,7 @@ class TaskManager: # @POST: Returns list of all Task objects. # @RETURN: List[Task] - All tasks. # @RELATION: [DEPENDS_ON] ->[TaskGraph] - def get_all_tasks(self) -> List[Task]: + def get_all_tasks(self) -> list[Task]: with belief_scope("TaskManager.get_all_tasks"): return list(self.tasks.values()) # #endregion get_all_tasks @@ -413,10 +416,10 @@ class TaskManager: self, limit: int = 10, offset: int = 0, - status: Optional[TaskStatus] = None, - plugin_ids: Optional[List[str]] = None, + status: TaskStatus | None = None, + plugin_ids: list[str] | None = None, completed_only: bool = False, - ) -> List[Task]: + ) -> list[Task]: with belief_scope("TaskManager.get_tasks"): tasks = list(self.tasks.values()) if status: @@ -436,7 +439,7 @@ class TaskManager: if not isinstance(started_at, datetime): return float("-inf") if started_at.tzinfo is None: - return started_at.replace(tzinfo=timezone.utc).timestamp() + return started_at.replace(tzinfo=UTC).timestamp() return started_at.timestamp() tasks.sort(key=sort_key, reverse=True) return tasks[offset : offset + limit] @@ -451,8 +454,8 @@ class TaskManager: # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs] # @RELATION: [DEPENDS_ON] ->[EventBus] def get_task_logs( - self, task_id: str, log_filter: Optional[LogFilter] = None - ) -> List[LogEntry]: + self, task_id: str, log_filter: LogFilter | None = None + ) -> list[LogEntry]: with belief_scope("TaskManager.get_task_logs", f"task_id={task_id}"): task = self.tasks.get(task_id) # For completed tasks, fetch from persistence @@ -494,7 +497,7 @@ class TaskManager: # @RETURN: List[str] - Unique source names. # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources] # @RELATION: [DEPENDS_ON] ->[EventBus] - def get_task_log_sources(self, task_id: str) -> List[str]: + def get_task_log_sources(self, task_id: str) -> list[str]: with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"): return self.log_persistence_service.get_sources(task_id) # #endregion get_task_log_sources @@ -516,8 +519,8 @@ class TaskManager: level: str, message: str, source: str = "system", - metadata: Optional[Dict[str, Any]] = None, - context: Optional[Dict[str, Any]] = None, + metadata: dict[str, Any] | None = None, + context: dict[str, Any] | None = None, ): with belief_scope("TaskManager._add_log", f"task_id={task_id}"): task = self.tasks.get(task_id) @@ -598,7 +601,7 @@ class TaskManager: # @THROWS: ValueError if task not found or not RUNNING. # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] # @RELATION: [DEPENDS_ON] ->[JobLifecycle] - def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None: + def await_input(self, task_id: str, input_request: dict[str, Any]) -> None: with belief_scope("TaskManager.await_input", f"task_id={task_id}"): task = self.tasks.get(task_id) if not task: @@ -628,7 +631,7 @@ class TaskManager: # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] # @RELATION: [DEPENDS_ON] ->[JobLifecycle] def resume_task_with_password( - self, task_id: str, passwords: Dict[str, str] + self, task_id: str, passwords: dict[str, str] ) -> None: with belief_scope( "TaskManager.resume_task_with_password", f"task_id={task_id}" @@ -665,7 +668,7 @@ class TaskManager: # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks] # @RELATION: [DEPENDS_ON] ->[TaskGraph] # @RELATION: [DEPENDS_ON] ->[EventBus] - def clear_tasks(self, status: Optional[TaskStatus] = None) -> int: + def clear_tasks(self, status: TaskStatus | None = None) -> int: with belief_scope("TaskManager.clear_tasks"): tasks_to_remove = [] for task_id, task in list(self.tasks.items()): @@ -702,4 +705,4 @@ class TaskManager: return len(tasks_to_remove) # #endregion clear_tasks # #endregion TaskManager -# #endregion TaskManagerModule \ No newline at end of file +# #endregion TaskManagerModule diff --git a/backend/src/core/task_manager/models.py b/backend/src/core/task_manager/models.py index 2b5430e1..d2c77364 100644 --- a/backend/src/core/task_manager/models.py +++ b/backend/src/core/task_manager/models.py @@ -8,8 +8,11 @@ import uuid from datetime import datetime from enum import Enum -from typing import Dict, Any, List, Optional +from typing import Any + from pydantic import BaseModel, Field + + # #region TaskStatus [TYPE Enum] class TaskStatus(str, Enum): PENDING = "PENDING" @@ -40,10 +43,10 @@ class LogEntry(BaseModel): source: str = Field( default="system" ) # Component attribution: plugin, superset_api, git, etc. - context: Optional[Dict[str, Any]] = ( + context: dict[str, Any] | None = ( None # Legacy field, kept for backward compatibility ) - metadata: Optional[Dict[str, Any]] = ( + metadata: dict[str, Any] | None = ( None # Structured metadata (e.g., dashboard_id, progress) ) # #endregion LogEntry @@ -57,15 +60,15 @@ class TaskLog(BaseModel): level: str source: str message: str - metadata: Optional[Dict[str, Any]] = None + metadata: dict[str, Any] | None = None class Config: from_attributes = True # #endregion TaskLog # #region LogFilter [TYPE Class] class LogFilter(BaseModel): - level: Optional[str] = None # Filter by log level - source: Optional[str] = None # Filter by source component - search: Optional[str] = None # Text search in message + level: str | None = None # Filter by log level + source: str | None = None # Filter by source component + search: str | None = None # Text search in message offset: int = Field(default=0, ge=0) limit: int = Field(default=100, ge=1, le=1000) # #endregion LogFilter @@ -74,8 +77,8 @@ class LogFilter(BaseModel): # @RELATION DEPENDS_ON -> [TaskLog] class LogStats(BaseModel): total_count: int - by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2} - by_source: Dict[str, int] # {"plugin": 5, "superset_api": 7} + by_level: dict[str, int] # {"INFO": 10, "ERROR": 2} + by_source: dict[str, int] # {"plugin": 5, "superset_api": 7} # #endregion LogStats # #region Task [C:3] [TYPE Class] [SEMANTICS task, job, execution, state, pydantic] # @BRIEF A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs. @@ -86,15 +89,15 @@ class Task(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4())) plugin_id: str status: TaskStatus = TaskStatus.PENDING - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - user_id: Optional[str] = None - logs: List[LogEntry] = Field(default_factory=list) - params: Dict[str, Any] = Field(default_factory=dict) + started_at: datetime | None = None + finished_at: datetime | None = None + user_id: str | None = None + logs: list[LogEntry] = Field(default_factory=list) + params: dict[str, Any] = Field(default_factory=dict) input_required: bool = False - input_request: Optional[Dict[str, Any]] = None + input_request: dict[str, Any] | None = None # Result payload can be dict/list/scalar depending on plugin and legacy records. - result: Optional[Any] = None + result: Any | None = None # #region __init__ [TYPE Function] # @PURPOSE: Initializes the Task model and validates input_request for AWAITING_INPUT status. # @PRE: If status is AWAITING_INPUT, input_request must be provided. @@ -106,4 +109,4 @@ class Task(BaseModel): raise ValueError("input_request is required when status is AWAITING_INPUT") # #endregion __init__ # #endregion Task -# #endregion TaskManagerModels \ No newline at end of file +# #endregion TaskManagerModels diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index a8a37c3d..00ed8715 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -9,16 +9,19 @@ # @RELATION DEPENDS_ON -> [TaskGraph] # @RELATION DEPENDS_ON -> [TasksSessionLocal] # @INVARIANT: Database schema must match the TaskRecord model structure. -from datetime import datetime -from typing import List, Optional import json import re +from datetime import datetime + from sqlalchemy.orm import Session -from ...models.task import TaskRecord, TaskLogRecord + from ...models.mapping import Environment +from ...models.task import TaskLogRecord, TaskRecord from ..database import TasksSessionLocal -from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats -from ..logger import logger, belief_scope +from ..logger import belief_scope, logger +from .models import LogEntry, LogFilter, LogStats, Task, TaskLog, TaskStatus + + # #region TaskPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, sqlalchemy] # @BRIEF Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models. # @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available. @@ -91,8 +94,8 @@ class TaskPersistenceService: # @RELATION: [DEPENDS_ON] ->[Environment] @staticmethod def _resolve_environment_id( - session: Session, env_id: Optional[str] - ) -> Optional[str]: + session: Session, env_id: str | None + ) -> str | None: with belief_scope("_resolve_environment_id"): raw_value = str(env_id or "").strip() if not raw_value: @@ -202,7 +205,7 @@ class TaskPersistenceService: # @POST: All tasks in list are persisted. # @PARAM: tasks (List[Task]) - The list of tasks to persist. # @RELATION: [CALLS] ->[persist_task] - def persist_tasks(self, tasks: List[Task]) -> None: + def persist_tasks(self, tasks: list[Task]) -> None: with belief_scope("TaskPersistenceService.persist_tasks"): for task in tasks: self.persist_task(task) @@ -218,8 +221,8 @@ class TaskPersistenceService: # @RELATION: [CALLS] ->[_json_load_if_needed] # @RELATION: [CALLS] ->[_parse_datetime] def load_tasks( - self, limit: int = 100, status: Optional[TaskStatus] = None - ) -> List[Task]: + self, limit: int = 100, status: TaskStatus | None = None + ) -> list[Task]: with belief_scope("TaskPersistenceService.load_tasks"): session: Session = TasksSessionLocal() try: @@ -272,7 +275,7 @@ class TaskPersistenceService: # @PARAM: task_ids (List[str]) - List of task IDs to delete. # @SIDE_EFFECT: Deletes rows from task_records table. # @RELATION: [DEPENDS_ON] ->[TaskRecord] - def delete_tasks(self, task_ids: List[str]) -> None: + def delete_tasks(self, task_ids: list[str]) -> None: if not task_ids: return with belief_scope("TaskPersistenceService.delete_tasks"): @@ -334,7 +337,7 @@ class TaskLogPersistenceService: # @SIDE_EFFECT: Writes to task_logs table. # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord] # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] - def add_logs(self, task_id: str, logs: List[LogEntry]) -> None: + def add_logs(self, task_id: str, logs: list[LogEntry]) -> None: if not logs: return with belief_scope("TaskLogPersistenceService.add_logs", f"task_id={task_id}"): @@ -370,7 +373,7 @@ class TaskLogPersistenceService: # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] # @RELATION: [DEPENDS_ON] ->[LogFilter] # @RELATION: [DEPENDS_ON] ->[TaskLog] - def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]: + def get_logs(self, task_id: str, log_filter: LogFilter) -> list[TaskLog]: with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"): session: Session = TasksSessionLocal() try: @@ -466,7 +469,7 @@ class TaskLogPersistenceService: # @RETURN: List[str] - Unique source names. # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]] # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] - def get_sources(self, task_id: str) -> List[str]: + def get_sources(self, task_id: str) -> list[str]: with belief_scope( "TaskLogPersistenceService.get_sources", f"task_id={task_id}" ): @@ -512,7 +515,7 @@ class TaskLogPersistenceService: # @PARAM: task_ids (List[str]) - List of task IDs. # @SIDE_EFFECT: Deletes rows from task_logs table. # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] - def delete_logs_for_tasks(self, task_ids: List[str]) -> None: + def delete_logs_for_tasks(self, task_ids: list[str]) -> None: if not task_ids: return with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"): @@ -529,4 +532,4 @@ class TaskLogPersistenceService: session.close() # #endregion delete_logs_for_tasks # #endregion TaskLogPersistenceService -# #endregion TaskPersistenceModule \ No newline at end of file +# #endregion TaskPersistenceModule diff --git a/backend/src/core/task_manager/task_logger.py b/backend/src/core/task_manager/task_logger.py index 93a38053..23f9b4c5 100644 --- a/backend/src/core/task_manager/task_logger.py +++ b/backend/src/core/task_manager/task_logger.py @@ -4,7 +4,10 @@ # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [EventBus] # @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source. -from typing import Dict, Any, Optional, Callable +from collections.abc import Callable +from typing import Any + + # #region TaskLogger [C:2] [TYPE Class] [SEMANTICS logger, task, source, attribution] # @BRIEF A wrapper around TaskManager._add_log that carries task_id and source context. # @RELATION DEPENDS_ON -> [TaskManager] @@ -74,8 +77,8 @@ class TaskLogger: self, level: str, message: str, - source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, + source: str | None = None, + metadata: dict[str, Any] | None = None, ) -> None: """Internal logging method.""" self._add_log( @@ -96,8 +99,8 @@ class TaskLogger: def debug( self, message: str, - source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, + source: str | None = None, + metadata: dict[str, Any] | None = None, ) -> None: self._log("DEBUG", message, source, metadata) # #endregion debug @@ -111,8 +114,8 @@ class TaskLogger: def info( self, message: str, - source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, + source: str | None = None, + metadata: dict[str, Any] | None = None, ) -> None: self._log("INFO", message, source, metadata) # #endregion info @@ -126,8 +129,8 @@ class TaskLogger: def warning( self, message: str, - source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, + source: str | None = None, + metadata: dict[str, Any] | None = None, ) -> None: self._log("WARNING", message, source, metadata) # #endregion warning @@ -141,8 +144,8 @@ class TaskLogger: def error( self, message: str, - source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None, + source: str | None = None, + metadata: dict[str, Any] | None = None, ) -> None: self._log("ERROR", message, source, metadata) # #endregion error @@ -154,11 +157,11 @@ class TaskLogger: # @PARAM: percent (float) - Progress percentage (0-100). # @PARAM: source (Optional[str]) - Override source. def progress( - self, message: str, percent: float, source: Optional[str] = None + self, message: str, percent: float, source: str | None = None ) -> None: """Log a progress update with percentage.""" metadata = {"progress": min(100, max(0, percent))} self._log("INFO", message, source, metadata) # #endregion progress # #endregion TaskLogger -# #endregion TaskLoggerModule \ No newline at end of file +# #endregion TaskLoggerModule diff --git a/backend/src/core/utils/async_network.py b/backend/src/core/utils/async_network.py index 45019814..5a3642e4 100644 --- a/backend/src/core/utils/async_network.py +++ b/backend/src/core/utils/async_network.py @@ -8,10 +8,13 @@ # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions] # @RELATION DEPENDS_ON -> [SupersetAuthCache] # @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401. -from typing import Optional, Dict, Any, Union import asyncio +from typing import Any + import httpx -from ..logger import logger as app_logger, belief_scope + +from ..logger import belief_scope +from ..logger import logger as app_logger from .network import ( AuthenticationError, DashboardNotFoundError, @@ -20,6 +23,8 @@ from .network import ( SupersetAPIError, SupersetAuthCache, ) + + # #region AsyncAPIClient [C:3] [TYPE Class] # @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache. # @RELATION DEPENDS_ON -> [SupersetAuthCache] @@ -27,7 +32,7 @@ from .network import ( # @RELATION CALLS -> [SupersetAuthCache.set] class AsyncAPIClient: DEFAULT_TIMEOUT = 30 - _auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {} + _auth_locks: dict[tuple[str, str, bool], asyncio.Lock] = {} # #region AsyncAPIClient.__init__ [TYPE Function] [C:3] # @PURPOSE: Initialize async API client for one environment. # @PRE: config contains base_url and auth payload. @@ -35,7 +40,7 @@ class AsyncAPIClient: # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str] # @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url] # @RELATION: [DEPENDS_ON] ->[SupersetAuthCache] - def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): + def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): self.base_url: str = self._normalize_base_url(config.get("base_url", "")) self.api_base_url: str = f"{self.base_url}/api/v1" self.auth = config.get("auth") @@ -45,7 +50,7 @@ class AsyncAPIClient: timeout=httpx.Timeout(timeout), follow_redirects=True, ) - self._tokens: Dict[str, str] = {} + self._tokens: dict[str, str] = {} self._authenticated = False self._auth_cache_key = SupersetAuthCache.build_key( self.base_url, @@ -95,7 +100,7 @@ class AsyncAPIClient: # @RELATION: [CALLS] ->[SupersetAuthCache.get] # @RELATION: [CALLS] ->[SupersetAuthCache.set] # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock] - async def authenticate(self) -> Dict[str, str]: + async def authenticate(self) -> dict[str, str]: cached_tokens = SupersetAuthCache.get(self._auth_cache_key) if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): self._tokens = cached_tokens @@ -148,7 +153,7 @@ class AsyncAPIClient: # @PURPOSE: Return authenticated Superset headers for async requests. # @POST: Headers include Authorization and CSRF tokens. # @RELATION: [CALLS] ->[AsyncAPIClient.authenticate] - async def get_headers(self) -> Dict[str, str]: + async def get_headers(self) -> dict[str, str]: if not self._authenticated: await self.authenticate() return { @@ -169,10 +174,10 @@ class AsyncAPIClient: self, method: str, endpoint: str, - headers: Optional[Dict[str, str]] = None, + headers: dict[str, str] | None = None, raw_response: bool = False, **kwargs, - ) -> Union[httpx.Response, Dict[str, Any]]: + ) -> httpx.Response | dict[str, Any]: full_url = self._build_api_url(endpoint) request_headers = await self.get_headers() if headers: @@ -262,4 +267,4 @@ class AsyncAPIClient: await self._client.aclose() # #endregion AsyncAPIClient.aclose # #endregion AsyncAPIClient -# #endregion AsyncNetworkModule \ No newline at end of file +# #endregion AsyncNetworkModule diff --git a/backend/src/core/utils/dataset_mapper.py b/backend/src/core/utils/dataset_mapper.py index d00acda1..482ca869 100644 --- a/backend/src/core/utils/dataset_mapper.py +++ b/backend/src/core/utils/dataset_mapper.py @@ -6,10 +6,15 @@ # @RELATION DEPENDS_ON -> pandas # @RELATION DEPENDS_ON -> psycopg2 # @PUBLIC_API: DatasetMapper +from typing import Any + import pandas as pd # type: ignore import psycopg2 # type: ignore -from typing import Dict, Optional, Any -from ..logger import logger as app_logger, belief_scope + +from ..logger import belief_scope +from ..logger import logger as app_logger + + # #region DatasetMapper [TYPE Class] # @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset. class DatasetMapper: @@ -29,7 +34,7 @@ class DatasetMapper: # @PARAM: table_name (str) - Имя таблицы. # @PARAM: table_schema (str) - Схема таблицы. # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам. - def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]: + def get_postgres_comments(self, db_config: dict, table_name: str, table_schema: str) -> dict[str, str]: with belief_scope("Fetch comments from PostgreSQL"): app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name) query = f""" @@ -90,7 +95,7 @@ class DatasetMapper: # @THROW: Exception - При ошибках чтения файла или парсинга. # @PARAM: file_path (str) - Путь к XLSX файлу. # @RETURN: Dict[str, str] - Словарь с меппингами. - def load_excel_mappings(self, file_path: str) -> Dict[str, str]: + def load_excel_mappings(self, file_path: str) -> dict[str, str]: with belief_scope("Load mappings from Excel"): app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path) try: @@ -118,11 +123,11 @@ class DatasetMapper: # @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу. # @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL. # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL. - def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None): + def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: dict | None = None, excel_path: str | None = None, table_name: str | None = None, table_schema: str | None = None): with belief_scope(f"Run dataset mapping for ID {dataset_id}"): app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source) - mappings: Dict[str, str] = {} - + mappings: dict[str, str] = {} + try: if source in ['postgres', 'both']: assert postgres_config and table_name and table_schema, "Postgres config is required." @@ -135,13 +140,13 @@ class DatasetMapper: return dataset_response = superset_client.get_dataset(dataset_id) dataset_data = dataset_response['result'] - + original_columns = dataset_data.get('columns', []) updated_columns = [] changes_made = False for column in original_columns: col_name = column.get('column_name') - + new_column = { "column_name": col_name, "id": column.get("id"), @@ -158,14 +163,14 @@ class DatasetMapper: "uuid": column.get("uuid"), "verbose_name": column.get("verbose_name"), } - + new_column = {k: v for k, v in new_column.items() if v is not None} if col_name in mappings: mapping_value = mappings[col_name] if isinstance(mapping_value, str) and new_column.get('verbose_name') != mapping_value: new_column['verbose_name'] = mapping_value changes_made = True - + updated_columns.append(new_column) updated_metrics = [] for metric in dataset_data.get("metrics", []): @@ -207,7 +212,7 @@ class DatasetMapper: "offset": dataset_data.get("offset"), "template_params": dataset_data.get("template_params"), } - + payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None} superset_client.update_dataset(dataset_id, payload_for_update) app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id) @@ -218,4 +223,4 @@ class DatasetMapper: return # #endregion run_mapping # #endregion DatasetMapper -# #endregion DatasetMapperModule \ No newline at end of file +# #endregion DatasetMapperModule diff --git a/backend/src/core/utils/fileio.py b/backend/src/core/utils/fileio.py index 538eb467..0c1e1351 100644 --- a/backend/src/core/utils/fileio.py +++ b/backend/src/core/utils/fileio.py @@ -7,16 +7,21 @@ # @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders import os import re -import zipfile -from pathlib import Path -from typing import Any, Optional, Tuple, Dict, List, Union, LiteralString, Generator -from contextlib import contextmanager -import tempfile -from datetime import date, datetime import shutil +import tempfile +import zipfile import zlib +from collections.abc import Generator +from contextlib import contextmanager from dataclasses import dataclass -from ..logger import logger as app_logger, belief_scope +from datetime import date, datetime +from pathlib import Path +from typing import Any, LiteralString + +from ..logger import belief_scope +from ..logger import logger as app_logger + + # #region InvalidZipFormatError [TYPE Class] # @BRIEF Exception raised when a file is not a valid ZIP archive. class InvalidZipFormatError(Exception): @@ -28,7 +33,7 @@ class InvalidZipFormatError(Exception): # @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста. # @YIELDS: Path - Путь к временному ресурсу. @contextmanager -def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]: +def create_temp_file(content: bytes | None = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path]: with belief_scope("Create temporary resource"): resource_path = None is_dir = suffix.startswith('.dir') @@ -84,7 +89,7 @@ def remove_empty_directories(root_dir: str) -> int: # @BRIEF Читает бинарное содержимое файла с диска. # @PRE: file_path должен указывать на существующий файл. # @POST: Возвращает байты содержимого и имя файла. -def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]: +def read_dashboard_from_disk(file_path: str) -> tuple[bytes, str]: with belief_scope(f"Read dashboard from {file_path}"): path = Path(file_path) assert path.is_file(), f"Файл дашборда не найден: {file_path}" @@ -99,10 +104,9 @@ def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]: # @PRE: file_path должен быть объектом Path к существующему файлу. # @POST: Возвращает 8-значную hex-строку CRC32. def calculate_crc32(file_path: Path) -> str: - with belief_scope(f"Calculate CRC32 for {file_path}"): - with open(file_path, 'rb') as f: - crc32_value = zlib.crc32(f.read()) - return f"{crc32_value:08x}" + with belief_scope(f"Calculate CRC32 for {file_path}"), open(file_path, 'rb') as f: + crc32_value = zlib.crc32(f.read()) + return f"{crc32_value:08x}" # #endregion calculate_crc32 # #region RetentionPolicy [TYPE DataClass] # @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные). @@ -125,7 +129,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir) return app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir) - + # 1. Collect all zip files zip_files = list(output_path.glob("*.zip")) if not zip_files: @@ -136,10 +140,10 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool app_logger.info("[archive_exports][State] Starting deduplication...") checksums = {} files_to_remove = [] - + # Sort by modification time (newest first) to keep the latest version zip_files.sort(key=lambda f: f.stat().st_mtime, reverse=True) - + for file_path in zip_files: try: crc = calculate_crc32(file_path) @@ -150,7 +154,7 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool checksums[crc] = file_path except Exception as e: app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e) - + for f in files_to_remove: try: f.unlink() @@ -171,14 +175,14 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool file_date = datetime.strptime(date_str, "%Y%m%d").date() except ValueError: pass - + if not file_date: # Fallback to modification time file_date = datetime.fromtimestamp(file_path.stat().st_mtime).date() - + files_with_dates.append((file_path, file_date)) files_to_keep = apply_retention_policy(files_with_dates, policy) - + for file_path, _ in files_with_dates: if file_path not in files_to_keep: try: @@ -191,30 +195,30 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool # @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить. # @PRE: files_with_dates is a list of (Path, date) tuples. # @POST: Returns a set of files to keep. -def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set: +def apply_retention_policy(files_with_dates: list[tuple[Path, date]], policy: RetentionPolicy) -> set: with belief_scope("Apply retention policy"): # Сортируем по дате (от новой к старой) - sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True) - # Словарь для хранения файлов по категориям - daily_files = [] - weekly_files = [] - monthly_files = [] - today = date.today() - for file_path, file_date in sorted_files: - # Ежедневные - if (today - file_date).days < policy.daily: - daily_files.append(file_path) - # Еженедельные - elif (today - file_date).days < policy.weekly * 7: - weekly_files.append(file_path) - # Ежемесячные - elif (today - file_date).days < policy.monthly * 30: - monthly_files.append(file_path) - # Возвращаем множество файлов, которые нужно сохранить - files_to_keep = set() - files_to_keep.update(daily_files) - files_to_keep.update(weekly_files[:policy.weekly]) - files_to_keep.update(monthly_files[:policy.monthly]) + sorted_files = sorted(files_with_dates, key=lambda x: x[1], reverse=True) + # Словарь для хранения файлов по категориям + daily_files = [] + weekly_files = [] + monthly_files = [] + today = date.today() + for file_path, file_date in sorted_files: + # Ежедневные + if (today - file_date).days < policy.daily: + daily_files.append(file_path) + # Еженедельные + elif (today - file_date).days < policy.weekly * 7: + weekly_files.append(file_path) + # Ежемесячные + elif (today - file_date).days < policy.monthly * 30: + monthly_files.append(file_path) + # Возвращаем множество файлов, которые нужно сохранить + files_to_keep = set() + files_to_keep.update(daily_files) + files_to_keep.update(weekly_files[:policy.weekly]) + files_to_keep.update(monthly_files[:policy.monthly]) app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep)) return files_to_keep # #endregion apply_retention_policy @@ -222,7 +226,7 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re # @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его. # @PRE: zip_content должен быть байтами валидного ZIP-архива. # @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir. -def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]: +def save_and_unpack_dashboard(zip_content: bytes, output_dir: str | Path, unpack: bool = False, original_filename: str | None = None) -> tuple[Path, Path | None]: with belief_scope("Save and unpack dashboard"): app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack) try: @@ -247,14 +251,14 @@ def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], # @PRE: path должен быть существующей директорией. # @POST: Все YAML файлы в директории обновлены согласно переданным параметрам. # @RELATION CALLS -> _update_yaml_file -def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None: +def update_yamls(db_configs: list[dict[str, Any]] | None = None, path: str = "dashboards", regexp_pattern: LiteralString | None = None, replace_string: LiteralString | None = None) -> None: with belief_scope("Update YAML configurations"): app_logger.info("[update_yamls][Enter] Starting YAML configuration update.") dir_path = Path(path) assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией" - - configs: List[Dict[str, Any]] = db_configs or [] - + + configs: list[dict[str, Any]] = db_configs or [] + for file_path in dir_path.rglob("*.yaml"): _update_yaml_file(file_path, configs, regexp_pattern, replace_string) # #endregion update_yamls @@ -262,11 +266,11 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = # @BRIEF (Helper) Обновляет один YAML файл. # @PRE: file_path должен быть объектом Path к существующему YAML файлу. # @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению. -def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None: +def _update_yaml_file(file_path: Path, db_configs: list[dict[str, Any]], regexp_pattern: str | None, replace_string: str | None) -> None: with belief_scope(f"Update YAML file: {file_path}"): # Читаем содержимое файла try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() except Exception as e: app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e) @@ -300,7 +304,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ val_pattern = re.escape(old_val) # Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц) pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)' - + # #region replacer [TYPE Function] # @PURPOSE: Функция замены, сохраняющая кавычки если они были. # @PRE: match должен быть объектом совпадения регулярного выражения. @@ -323,7 +327,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ # @BRIEF Создает ZIP-архив из указанных исходных путей. # @PRE: source_paths должен содержать существующие пути. # @POST: ZIP-архив создан по пути zip_path. -def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool: +def create_dashboard_export(zip_path: str | Path, source_paths: list[str | Path], exclude_extensions: list[str] | None = None) -> bool: with belief_scope(f"Create dashboard export: {zip_path}"): app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path) try: @@ -338,7 +342,7 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union zipf.write(item, arcname) app_logger.reflect(f"Archive created: {zip_path}", extra={"src": "create_dashboard_export"}) return True - except (IOError, zipfile.BadZipFile, AssertionError) as e: + except (OSError, zipfile.BadZipFile, AssertionError) as e: app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True) return False # #endregion create_dashboard_export @@ -354,7 +358,7 @@ def sanitize_filename(filename: str) -> str: # @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'. # @PRE: headers должен быть словарем заголовков. # @POST: Возвращает имя файла или None, если заголовок отсутствует. -def get_filename_from_headers(headers: dict) -> Optional[str]: +def get_filename_from_headers(headers: dict) -> str | None: with belief_scope("Get filename from headers"): content_disposition = headers.get("Content-Disposition", "") if match := re.search(r'filename="?([^"]+)"?', content_disposition): @@ -367,51 +371,51 @@ def get_filename_from_headers(headers: dict) -> Optional[str]: # @POST: Директории с одинаковым префиксом объединены в одну. def consolidate_archive_folders(root_directory: Path) -> None: with belief_scope(f"Consolidate archives in {root_directory}"): - assert isinstance(root_directory, Path), "root_directory must be a Path object." - assert root_directory.is_dir(), "root_directory must be an existing directory." - - app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory) - # Собираем все директории с архивами - archive_dirs = [] - for item in root_directory.iterdir(): - if item.is_dir(): - # Проверяем, есть ли в директории ZIP-архивы - if any(item.glob("*.zip")): - archive_dirs.append(item) - # Группируем по слагу (части имени до первого '_') - slug_groups = {} - for dir_path in archive_dirs: - dir_name = dir_path.name - slug = dir_name.split('_')[0] if '_' in dir_name else dir_name - if slug not in slug_groups: - slug_groups[slug] = [] - slug_groups[slug].append(dir_path) - # Для каждой группы консолидируем - for slug, dirs in slug_groups.items(): - if len(dirs) <= 1: - continue - # Создаем целевую директорию - target_dir = root_directory / slug - target_dir.mkdir(exist_ok=True) - app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir) - # Перемещаем содержимое - for source_dir in dirs: - if source_dir == target_dir: - continue - for item in source_dir.iterdir(): - dest_item = target_dir / item.name - try: - if item.is_dir(): - shutil.move(str(item), str(dest_item)) - else: - shutil.move(str(item), str(dest_item)) - except Exception as e: - app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e) - # Удаляем исходную директорию - try: - source_dir.rmdir() - app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir) - except Exception as e: - app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e) + assert isinstance(root_directory, Path), "root_directory must be a Path object." + assert root_directory.is_dir(), "root_directory must be an existing directory." + + app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory) + # Собираем все директории с архивами + archive_dirs = [] + for item in root_directory.iterdir(): + if item.is_dir(): + # Проверяем, есть ли в директории ZIP-архивы + if any(item.glob("*.zip")): + archive_dirs.append(item) + # Группируем по слагу (части имени до первого '_') + slug_groups = {} + for dir_path in archive_dirs: + dir_name = dir_path.name + slug = dir_name.split('_')[0] if '_' in dir_name else dir_name + if slug not in slug_groups: + slug_groups[slug] = [] + slug_groups[slug].append(dir_path) + # Для каждой группы консолидируем + for slug, dirs in slug_groups.items(): + if len(dirs) <= 1: + continue + # Создаем целевую директорию + target_dir = root_directory / slug + target_dir.mkdir(exist_ok=True) + app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir) + # Перемещаем содержимое + for source_dir in dirs: + if source_dir == target_dir: + continue + for item in source_dir.iterdir(): + dest_item = target_dir / item.name + try: + if item.is_dir(): + shutil.move(str(item), str(dest_item)) + else: + shutil.move(str(item), str(dest_item)) + except Exception as e: + app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e) + # Удаляем исходную директорию + try: + source_dir.rmdir() + app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir) + except Exception as e: + app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e) # #endregion consolidate_archive_folders -# #endregion FileIO \ No newline at end of file +# #endregion FileIO diff --git a/backend/src/core/utils/matching.py b/backend/src/core/utils/matching.py index 9b35cec5..53baa47a 100644 --- a/backend/src/core/utils/matching.py +++ b/backend/src/core/utils/matching.py @@ -6,14 +6,15 @@ # # @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0. + from rapidfuzz import fuzz, process -from typing import List, Dict + # #region suggest_mappings [TYPE Function] # @BRIEF Suggests mappings between source and target databases using fuzzy matching. # @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'. # @POST: Returns a list of suggested mappings with confidence scores. -def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], threshold: int = 60) -> List[Dict]: +def suggest_mappings(source_databases: list[dict], target_databases: list[dict], threshold: int = 60) -> list[dict]: """ Suggest mappings between source and target databases using fuzzy matching. """ @@ -22,15 +23,15 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], return suggestions target_names = [db['database_name'] for db in target_databases] - + for s_db in source_databases: # Use token_sort_ratio as decided in research.md match = process.extractOne( - s_db['database_name'], - target_names, + s_db['database_name'], + target_names, scorer=fuzz.token_sort_ratio ) - + if match: name, score, index = match if score >= threshold: @@ -41,7 +42,7 @@ def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], "target_db_uuid": target_databases[index]['uuid'], "confidence": score / 100.0 }) - + return suggestions # #endregion suggest_mappings diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index a4671a44..900133bc 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -4,17 +4,22 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> [LoggerModule] # @PUBLIC_API: APIClient -from typing import Optional, Dict, Any, List, Union, cast, Tuple -import json import io -from pathlib import Path +import json import threading import time +from pathlib import Path +from typing import Any, cast + import requests -from requests.adapters import HTTPAdapter import urllib3 +from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -from ..logger import logger as app_logger, belief_scope + +from ..logger import belief_scope +from ..logger import logger as app_logger + + # #region SupersetAPIError [C:1] [TYPE Class] # @BRIEF Base exception for all Superset API related errors. class SupersetAPIError(Exception): @@ -59,7 +64,7 @@ class DashboardNotFoundError(SupersetAPIError): # @PURPOSE: Initializes the not found error with resource ID. # @PRE: resource_id is provided. # @POST: DashboardNotFoundError is initialized. - def __init__(self, resource_id: Union[int, str], message: str = "Dashboard not found", **context: Any): + def __init__(self, resource_id: int | str, message: str = "Dashboard not found", **context: Any): with belief_scope("DashboardNotFoundError.__init__"): super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context) # #endregion __init__ @@ -84,16 +89,16 @@ class NetworkError(Exception): class SupersetAuthCache: TTL_SECONDS = 300 _lock = threading.Lock() - _entries: Dict[Tuple[str, str, bool], Dict[str, Any]] = {} + _entries: dict[tuple[str, str, bool], dict[str, Any]] = {} @classmethod - def build_key(cls, base_url: str, auth: Optional[Dict[str, Any]], verify_ssl: bool) -> Tuple[str, str, bool]: + def build_key(cls, base_url: str, auth: dict[str, Any] | None, verify_ssl: bool) -> tuple[str, str, bool]: username = "" if isinstance(auth, dict): username = str(auth.get("username") or "").strip() return (str(base_url or "").strip(), username, bool(verify_ssl)) @classmethod # #region SupersetAuthCache.get [TYPE Function] - def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]: + def get(cls, key: tuple[str, str, bool]) -> dict[str, str] | None: now = time.time() with cls._lock: payload = cls._entries.get(key) @@ -114,7 +119,7 @@ class SupersetAuthCache: # #endregion SupersetAuthCache.get @classmethod # #region SupersetAuthCache.set [TYPE Function] - def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None: + def set(cls, key: tuple[str, str, bool], tokens: dict[str, str], ttl_seconds: int | None = None) -> None: normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1) with cls._lock: cls._entries[key] = { @@ -126,7 +131,7 @@ class SupersetAuthCache: } # #endregion SupersetAuthCache.set @classmethod - def invalidate(cls, key: Tuple[str, str, bool]) -> None: + def invalidate(cls, key: tuple[str, str, bool]) -> None: with cls._lock: cls._entries.pop(key, None) # #endregion SupersetAuthCache @@ -143,7 +148,7 @@ class APIClient: # @PARAM: timeout (int) - Таймаут запросов. # @PRE: config must contain 'base_url' and 'auth'. # @POST: APIClient instance is initialized with a session. - def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): + def __init__(self, config: dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): with belief_scope("__init__"): app_logger.reason("Initializing APIClient.", extra={"src": "APIClient.__init__"}) self.base_url: str = self._normalize_base_url(config.get("base_url", "")) @@ -151,7 +156,7 @@ class APIClient: self.auth = config.get("auth") self.request_settings = {"verify_ssl": verify_ssl, "timeout": timeout} self.session = self._init_session() - self._tokens: Dict[str, str] = {} + self._tokens: dict[str, str] = {} self._auth_cache_key = SupersetAuthCache.build_key( self.base_url, self.auth, @@ -168,33 +173,34 @@ class APIClient: def _init_session(self) -> requests.Session: with belief_scope("_init_session"): session = requests.Session() - + # Create a custom adapter that handles TLS issues class TLSAdapter(HTTPAdapter): def init_poolmanager(self, connections, maxsize, block=False): - from urllib3.poolmanager import PoolManager import ssl - + + from urllib3.poolmanager import PoolManager + # Create an SSL context that ignores TLSv1 unrecognized name errors ctx = ssl.create_default_context() ctx.set_ciphers('HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA') - + # Ignore TLSV1_UNRECOGNIZED_NAME errors by disabling hostname verification # This is safe when verify_ssl is false (we're already not verifying the certificate) ctx.check_hostname = False - + self.poolmanager = PoolManager( num_pools=connections, maxsize=maxsize, block=block, ssl_context=ctx ) - + retries = Retry(total=3, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]) adapter = TLSAdapter(max_retries=retries) session.mount('http://', adapter) session.mount('https://', adapter) - + if not self.request_settings["verify_ssl"]: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) app_logger.warning("[_init_session][State] SSL verification disabled.") @@ -202,7 +208,7 @@ class APIClient: session.verify = False else: session.verify = True - + return session # #endregion _init_session # #region _normalize_base_url [TYPE Function] @@ -239,7 +245,7 @@ class APIClient: # @THROW: AuthenticationError, NetworkError - при ошибках. # @RELATION: [CALLS] ->[SupersetAuthCache.get] # @RELATION: [CALLS] ->[SupersetAuthCache.set] - def authenticate(self) -> Dict[str, str]: + def authenticate(self) -> dict[str, str]: with belief_scope("authenticate"): app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url) cached_tokens = SupersetAuthCache.get(self._auth_cache_key) @@ -288,19 +294,19 @@ class APIClient: masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()} app_logger.info(f"[authenticate][Debug] Login URL: {login_url}") app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}") - + response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"]) - + if response.status_code != 200: app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}") - + response.raise_for_status() access_token = response.json()["access_token"] - + csrf_url = f"{self.api_base_url}/security/csrf_token/" csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"]) csrf_response.raise_for_status() - + self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]} self._authenticated = True SupersetAuthCache.set(self._auth_cache_key, self._tokens) @@ -321,7 +327,7 @@ class APIClient: # @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов. # @PRE: APIClient is initialized and authenticated or can be authenticated. # @POST: Returns headers including auth tokens. - def headers(self) -> Dict[str, str]: + def headers(self) -> dict[str, str]: if not self._authenticated: self.authenticate() return { @@ -341,12 +347,12 @@ class APIClient: # @POST: Returns response content or raw Response object. # @RETURN: `requests.Response` если `raw_response=True`, иначе `dict`. # @THROW: SupersetAPIError, NetworkError и их подклассы. - def request(self, method: str, endpoint: str, headers: Optional[Dict] = None, raw_response: bool = False, **kwargs) -> Union[requests.Response, Dict[str, Any]]: + def request(self, method: str, endpoint: str, headers: dict | None = None, raw_response: bool = False, **kwargs) -> requests.Response | dict[str, Any]: full_url = self._build_api_url(endpoint) _headers = self.headers.copy() if headers: _headers.update(headers) - + try: response = self.session.request(method, full_url, headers=_headers, **kwargs) response.raise_for_status() @@ -429,15 +435,15 @@ class APIClient: # @POST: File is uploaded and response returned. # @RETURN: Ответ API в виде словаря. # @THROW: SupersetAPIError, NetworkError, TypeError. - def upload_file(self, endpoint: str, file_info: Dict[str, Any], extra_data: Optional[Dict] = None, timeout: Optional[int] = None) -> Dict: + def upload_file(self, endpoint: str, file_info: dict[str, Any], extra_data: dict | None = None, timeout: int | None = None) -> dict: with belief_scope("upload_file"): full_url = self._build_api_url(endpoint) _headers = self.headers.copy() _headers.pop('Content-Type', None) - - + + file_obj, file_name, form_field = file_info.get("file_obj"), file_info.get("file_name"), file_info.get("form_field", "file") - + files_payload = {} if isinstance(file_obj, (str, Path)): with open(file_obj, 'rb') as f: @@ -446,7 +452,7 @@ class APIClient: files_payload = {form_field: (file_name, file_obj.getvalue(), 'application/x-zip-compressed')} else: raise TypeError(f"Unsupported file_obj type: {type(file_obj)}") - + return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout) # #endregion upload_file # #region _perform_upload [TYPE Function] @@ -459,7 +465,7 @@ class APIClient: # @PRE: url, files, and headers must be provided. # @POST: POST request is performed and JSON response returned. # @RETURN: Dict - Ответ. - def _perform_upload(self, url: str, files: Dict, data: Optional[Dict], headers: Dict, timeout: Optional[int]) -> Dict: + def _perform_upload(self, url: str, files: dict, data: dict | None, headers: dict, timeout: int | None) -> dict: with belief_scope("_perform_upload"): try: response = self.session.post(url, files=files, data=data or {}, headers=headers, timeout=timeout or self.request_settings["timeout"]) @@ -484,9 +490,9 @@ class APIClient: # @PRE: query_params must be a dictionary. # @POST: Returns total count of items. # @RETURN: int - Количество. - def fetch_paginated_count(self, endpoint: str, query_params: Dict, count_field: str = "count") -> int: + def fetch_paginated_count(self, endpoint: str, query_params: dict, count_field: str = "count") -> int: with belief_scope("fetch_paginated_count"): - response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)})) + response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)})) return response_json.get(count_field, 0) # #endregion fetch_paginated_count # #region fetch_paginated_data [TYPE Function] @@ -496,26 +502,26 @@ class APIClient: # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional. # @POST: Returns all items across all pages. # @RETURN: List[Any] - Список данных. - def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]: + def fetch_paginated_data(self, endpoint: str, pagination_options: dict[str, Any]) -> list[Any]: with belief_scope("fetch_paginated_data"): base_query = pagination_options["base_query"] total_count = pagination_options.get("total_count") - + results_field = pagination_options["results_field"] count_field = pagination_options.get("count_field", "count") page_size = base_query.get('page_size', 1000) assert page_size > 0, "'page_size' must be a positive number." - + results = [] page = 0 - + # Fetch first page to get data and total count if not provided query = {**base_query, 'page': page} - response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)})) - + response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)})) + first_page_results = response_json.get(results_field, []) results.extend(first_page_results) - + if total_count is None: total_count = response_json.get(count_field, len(first_page_results)) app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}") @@ -523,10 +529,10 @@ class APIClient: total_pages = (total_count + page_size - 1) // page_size for page in range(1, total_pages): query = {**base_query, 'page': page} - response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)})) + response_json = cast(dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)})) results.extend(response_json.get(results_field, [])) - + return results # #endregion fetch_paginated_data # #endregion APIClient -# #endregion NetworkModule \ No newline at end of file +# #endregion NetworkModule diff --git a/backend/src/core/utils/superset_compilation_adapter.py b/backend/src/core/utils/superset_compilation_adapter.py index 3775f15d..f745e3f0 100644 --- a/backend/src/core/utils/superset_compilation_adapter.py +++ b/backend/src/core/utils/superset_compilation_adapter.py @@ -8,14 +8,18 @@ # @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls. # @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only. from __future__ import annotations + # #region SupersetCompilationAdapter.imports [TYPE Block] from dataclasses import dataclass from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any + from src.core.config_models import Environment from src.core.logger import belief_scope, logger from src.core.superset_client import SupersetClient from src.models.dataset_review import CompiledPreview, PreviewStatus + + # #endregion SupersetCompilationAdapter.imports # #region PreviewCompilationPayload [C:2] [TYPE Class] # @BRIEF Typed preview payload for Superset-side compilation. @@ -24,8 +28,8 @@ class PreviewCompilationPayload: session_id: str dataset_id: int preview_fingerprint: str - template_params: Dict[str, Any] - effective_filters: List[Dict[str, Any]] + template_params: dict[str, Any] + effective_filters: list[dict[str, Any]] # #endregion PreviewCompilationPayload # #region SqlLabLaunchPayload [C:2] [TYPE Class] # @BRIEF Typed SQL Lab payload for audited launch handoff. @@ -35,7 +39,7 @@ class SqlLabLaunchPayload: dataset_id: int preview_id: str compiled_sql: str - template_params: Dict[str, Any] + template_params: dict[str, Any] # #endregion SqlLabLaunchPayload # #region SupersetCompilationAdapter [C:4] [TYPE Class] # @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication. @@ -47,7 +51,7 @@ class SupersetCompilationAdapter: # #region SupersetCompilationAdapter.__init__ [TYPE Function] [C:2] # @PURPOSE: Bind adapter to one Superset environment and client instance. def __init__( - self, environment: Environment, client: Optional[SupersetClient] = None + self, environment: Environment, client: SupersetClient | None = None ) -> None: self.environment = environment self.client = client or SupersetClient(environment) @@ -221,7 +225,7 @@ class SupersetCompilationAdapter: # @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]] def _request_superset_preview( self, payload: PreviewCompilationPayload - ) -> Dict[str, Any]: + ) -> dict[str, Any]: direct_compile_preview = getattr(self.client, "compile_preview", None) if self._supports_client_method("compile_preview") and callable( direct_compile_preview @@ -313,7 +317,7 @@ class SupersetCompilationAdapter: "Superset preview compilation response could not be normalized" ) return normalized - errors: List[str] = [] + errors: list[str] = [] for endpoint in ( f"/dataset/{payload.dataset_id}/preview", f"/dataset/{payload.dataset_id}/sql", @@ -357,7 +361,7 @@ class SupersetCompilationAdapter: # @POST: returns the first successful SQL Lab execution response from Superset. # @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests. # @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]] - def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]: + def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> dict[str, Any]: dataset_raw = self.client.get_dataset(payload.dataset_id) dataset_record = ( dataset_raw.get("result", dataset_raw) @@ -384,7 +388,7 @@ class SupersetCompilationAdapter: {"kind": "network", "target": "/sqllab/execute/", "http_method": "POST"}, {"kind": "network", "target": "/sql_lab/execute/", "http_method": "POST"}, ] - errors: List[str] = [] + errors: list[str] = [] for candidate in candidate_calls: try: response = self.client.network.request( @@ -408,7 +412,7 @@ class SupersetCompilationAdapter: # #region SupersetCompilationAdapter._normalize_preview_response [TYPE Function] [C:3] # @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure. # @RELATION: [DEPENDS_ON] ->[CompiledPreview] - def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]: + def _normalize_preview_response(self, response: Any) -> dict[str, Any] | None: if not isinstance(response, dict): return None compiled_sql_candidates = [ @@ -436,9 +440,9 @@ class SupersetCompilationAdapter: # #endregion SupersetCompilationAdapter._normalize_preview_response # #region SupersetCompilationAdapter._dump_json [TYPE Function] [C:1] # @PURPOSE: Serialize Superset request payload deterministically for network transport. - def _dump_json(self, payload: Dict[str, Any]) -> str: + def _dump_json(self, payload: dict[str, Any]) -> str: import json return json.dumps(payload, sort_keys=True, default=str) # #endregion SupersetCompilationAdapter._dump_json # #endregion SupersetCompilationAdapter -# #endregion SupersetCompilationAdapter \ No newline at end of file +# #endregion SupersetCompilationAdapter diff --git a/backend/src/core/utils/superset_context_extractor/__init__.py b/backend/src/core/utils/superset_context_extractor/__init__.py index 9109c458..892e36e8 100644 --- a/backend/src/core/utils/superset_context_extractor/__init__.py +++ b/backend/src/core/utils/superset_context_extractor/__init__.py @@ -17,11 +17,11 @@ # #endregion SupersetContextExtractorPackage from ._base import SupersetContextExtractorBase, SupersetParsedContext +from ._filters import SupersetContextFiltersExtractMixin from ._parsing import SupersetContextParsingMixin +from ._pii import _mask_sensitive_text, mask_pii_value, sanitize_imported_filter_for_assistant from ._recovery import SupersetContextRecoveryMixin from ._templates import SupersetContextTemplatesMixin -from ._filters import SupersetContextFiltersExtractMixin -from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_sensitive_text # #region SupersetContextExtractor [C:4] [TYPE Class] @@ -57,7 +57,7 @@ class SupersetContextExtractor( __all__ = [ "SupersetContextExtractor", "SupersetParsedContext", + "_mask_sensitive_text", "mask_pii_value", "sanitize_imported_filter_for_assistant", - "_mask_sensitive_text", ] diff --git a/backend/src/core/utils/superset_context_extractor/_base.py b/backend/src/core/utils/superset_context_extractor/_base.py index c2179c17..9dd41c79 100644 --- a/backend/src/core/utils/superset_context_extractor/_base.py +++ b/backend/src/core/utils/superset_context_extractor/_base.py @@ -11,15 +11,16 @@ # #endregion SupersetContextExtractorBase # #region _base_imports [TYPE Block] from __future__ import annotations + import json -import re -from copy import deepcopy from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set, cast -from urllib.parse import parse_qs, unquote, urlparse +from typing import Any, cast +from urllib.parse import unquote + from ...config_models import Environment -from ...logger import belief_scope, logger +from ...logger import logger from ...superset_client import SupersetClient + # #endregion _base_imports logger = cast(Any, logger) # #region SupersetParsedContext [C:2] [TYPE Class] @@ -28,15 +29,15 @@ logger = cast(Any, logger) class SupersetParsedContext: source_url: str dataset_ref: str - dataset_id: Optional[int] = None - dashboard_id: Optional[int] = None - chart_id: Optional[int] = None + dataset_id: int | None = None + dashboard_id: int | None = None + chart_id: int | None = None resource_type: str = "unknown" - query_state: Dict[str, Any] = field(default_factory=dict) - imported_filters: List[Dict[str, Any]] = field(default_factory=list) - unresolved_references: List[str] = field(default_factory=list) + query_state: dict[str, Any] = field(default_factory=dict) + imported_filters: list[dict[str, Any]] = field(default_factory=list) + unresolved_references: list[str] = field(default_factory=list) partial_recovery: bool = False - dataset_payload: Optional[Dict[str, Any]] = None + dataset_payload: dict[str, Any] | None = None # #endregion SupersetParsedContext # #region SupersetContextExtractorBase [C:4] [TYPE Class] # @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers. @@ -48,7 +49,7 @@ class SupersetContextExtractorBase: # #region SupersetContextExtractorBase.__init__ [TYPE Function] [C:2] # @PURPOSE: Bind extractor to one Superset environment and client instance. def __init__( - self, environment: Environment, client: Optional[SupersetClient] = None + self, environment: Environment, client: SupersetClient | None = None ) -> None: self.environment = environment self.client = client or SupersetClient(environment) @@ -57,7 +58,7 @@ class SupersetContextExtractorBase: # @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX. def build_recovery_summary( self, parsed_context: SupersetParsedContext - ) -> Dict[str, Any]: + ) -> dict[str, Any]: return { "dataset_ref": parsed_context.dataset_ref, "dataset_id": parsed_context.dataset_id, @@ -71,8 +72,8 @@ class SupersetContextExtractorBase: # #region SupersetContextExtractorBase._extract_numeric_identifier [TYPE Function] [C:2] # @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path. def _extract_numeric_identifier( - self, path_parts: List[str], resource_name: str - ) -> Optional[int]: + self, path_parts: list[str], resource_name: str + ) -> int | None: if resource_name not in path_parts: return None try: @@ -88,7 +89,7 @@ class SupersetContextExtractorBase: # #endregion SupersetContextExtractorBase._extract_numeric_identifier # #region SupersetContextExtractorBase._extract_dashboard_reference [TYPE Function] [C:2] # @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path. - def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]: + def _extract_dashboard_reference(self, path_parts: list[str]) -> str | None: if "dashboard" not in path_parts: return None try: @@ -104,7 +105,7 @@ class SupersetContextExtractorBase: # #endregion SupersetContextExtractorBase._extract_dashboard_reference # #region SupersetContextExtractorBase._extract_dashboard_permalink_key [TYPE Function] [C:2] # @PURPOSE: Extract a dashboard permalink key from a Superset URL path. - def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]: + def _extract_dashboard_permalink_key(self, path_parts: list[str]) -> str | None: if "dashboard" not in path_parts: return None try: @@ -121,7 +122,7 @@ class SupersetContextExtractorBase: # #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key # #region SupersetContextExtractorBase._extract_dashboard_id_from_state [TYPE Function] [C:2] # @PURPOSE: Extract a dashboard identifier from returned permalink state when present. - def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: + def _extract_dashboard_id_from_state(self, state: dict[str, Any]) -> int | None: return self._search_nested_numeric_key( payload=state, candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"}, @@ -129,7 +130,7 @@ class SupersetContextExtractorBase: # #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state # #region SupersetContextExtractorBase._extract_chart_id_from_state [TYPE Function] [C:2] # @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent. - def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: + def _extract_chart_id_from_state(self, state: dict[str, Any]) -> int | None: return self._search_nested_numeric_key( payload=state, candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"}, @@ -139,8 +140,8 @@ class SupersetContextExtractorBase: # @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set. # @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase] def _search_nested_numeric_key( - self, payload: Any, candidate_keys: Set[str] - ) -> Optional[int]: + self, payload: Any, candidate_keys: set[str] + ) -> int | None: if isinstance(payload, dict): for key, value in payload.items(): if key in candidate_keys: @@ -161,8 +162,8 @@ class SupersetContextExtractorBase: # #endregion SupersetContextExtractorBase._search_nested_numeric_key # #region SupersetContextExtractorBase._decode_query_state [TYPE Function] [C:2] # @PURPOSE: Decode query-string structures used by Superset URL state transport. - def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]: - query_state: Dict[str, Any] = {} + def _decode_query_state(self, query_params: dict[str, list[str]]) -> dict[str, Any]: + query_state: dict[str, Any] = {} for key, values in query_params.items(): if not values: continue @@ -180,4 +181,4 @@ class SupersetContextExtractorBase: query_state[key] = decoded_value return query_state # #endregion SupersetContextExtractorBase._decode_query_state -# #endregion SupersetContextExtractorBase \ No newline at end of file +# #endregion SupersetContextExtractorBase diff --git a/backend/src/core/utils/superset_context_extractor/_filters.py b/backend/src/core/utils/superset_context_extractor/_filters.py index a4f66c0c..9359d3e1 100644 --- a/backend/src/core/utils/superset_context_extractor/_filters.py +++ b/backend/src/core/utils/superset_context_extractor/_filters.py @@ -5,9 +5,12 @@ # #endregion SupersetContextFiltersExtractMixin # #region _filters_imports [TYPE Block] from __future__ import annotations + from copy import deepcopy -from typing import Any, Dict, List, cast +from typing import Any, cast + from ...logger import logger as app_logger + app_logger = cast(Any, app_logger) # #endregion _filters_imports # #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class] @@ -16,9 +19,9 @@ class SupersetContextFiltersExtractMixin: # #region SupersetContextFiltersExtractMixin._extract_imported_filters [TYPE Function] [C:2] # @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values. def _extract_imported_filters( - self, query_state: Dict[str, Any] - ) -> List[Dict[str, Any]]: - imported_filters: List[Dict[str, Any]] = [] + self, query_state: dict[str, Any] + ) -> list[dict[str, Any]]: + imported_filters: list[dict[str, Any]] = [] native_filters_payload = query_state.get("native_filters") if isinstance(native_filters_payload, list): for index, item in enumerate(native_filters_payload): @@ -230,4 +233,4 @@ class SupersetContextFiltersExtractMixin: ) return imported_filters # #endregion SupersetContextFiltersExtractMixin._extract_imported_filters -# #endregion SupersetContextFiltersExtractMixin \ No newline at end of file +# #endregion SupersetContextFiltersExtractMixin diff --git a/backend/src/core/utils/superset_context_extractor/_parsing.py b/backend/src/core/utils/superset_context_extractor/_parsing.py index 8be0e3c8..492ba93c 100644 --- a/backend/src/core/utils/superset_context_extractor/_parsing.py +++ b/backend/src/core/utils/superset_context_extractor/_parsing.py @@ -6,10 +6,13 @@ # #endregion SupersetContextParsingMixin # #region _parsing_imports [TYPE Block] from __future__ import annotations -from typing import Any, Dict, List, Optional, cast + +from typing import Any, cast from urllib.parse import parse_qs, urlparse + from ...logger import belief_scope, logger from ._base import SupersetParsedContext + logger = cast(Any, logger) # #endregion _parsing_imports # #region SupersetContextParsingMixin [C:4] [TYPE Class] @@ -48,9 +51,9 @@ class SupersetContextParsingMixin: dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts) chart_id = self._extract_numeric_identifier(path_parts, "chart") resource_type = "unknown" - dataset_ref: Optional[str] = None + dataset_ref: str | None = None partial_recovery = False - unresolved_references: List[str] = [] + unresolved_references: list[str] = [] if dataset_id is not None: resource_type = "dataset" dataset_ref = f"dataset:{dataset_id}" @@ -253,7 +256,7 @@ class SupersetContextParsingMixin: extra={"path": parsed_url.path}, ) raise ValueError("Unsupported Superset link shape") - dataset_payload: Optional[Dict[str, Any]] = None + dataset_payload: dict[str, Any] | None = None if dataset_id is not None: try: dataset_payload = self.client.get_dataset_detail(dataset_id) @@ -317,9 +320,9 @@ class SupersetContextParsingMixin: def _recover_dataset_binding_from_dashboard( self, dashboard_id: int, - dataset_ref: Optional[str], - unresolved_references: List[str], - ) -> tuple[Optional[int], List[str]]: + dataset_ref: str | None, + unresolved_references: list[str], + ) -> tuple[int | None, list[str]]: dashboard_detail = self.client.get_dashboard_detail(dashboard_id) datasets = dashboard_detail.get("datasets") or [] if datasets: @@ -349,4 +352,4 @@ class SupersetContextParsingMixin: unresolved_references.append("dashboard_dataset_binding_missing") return None, unresolved_references # #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard -# #endregion SupersetContextParsingMixin \ No newline at end of file +# #endregion SupersetContextParsingMixin diff --git a/backend/src/core/utils/superset_context_extractor/_pii.py b/backend/src/core/utils/superset_context_extractor/_pii.py index 9320b81f..9e9bb3af 100644 --- a/backend/src/core/utils/superset_context_extractor/_pii.py +++ b/backend/src/core/utils/superset_context_extractor/_pii.py @@ -6,7 +6,8 @@ # #region _pii_imports [TYPE Block] import re from copy import deepcopy -from typing import Any, Dict, List, Tuple +from typing import Any + # #endregion _pii_imports _EMAIL_PATTERN = re.compile( @@ -23,12 +24,12 @@ _MIXED_IDENTIFIER_PATTERN = re.compile( # #region mask_pii_value [C:2] [TYPE Function] # @BRIEF Redact likely sensitive identifiers while preserving structural hints for assistant-facing dataset-review context. -def mask_pii_value(value: Any) -> Tuple[Any, bool]: +def mask_pii_value(value: Any) -> tuple[Any, bool]: if isinstance(value, str): return _mask_sensitive_text(value) if isinstance(value, list): masked_any = False - masked_list: List[Any] = [] + masked_list: list[Any] = [] for item in value: masked_item, item_masked = mask_pii_value(item) masked_list.append(masked_item) @@ -36,7 +37,7 @@ def mask_pii_value(value: Any) -> Tuple[Any, bool]: return masked_list, masked_any if isinstance(value, dict): masked_any = False - masked_dict: Dict[str, Any] = {} + masked_dict: dict[str, Any] = {} for key, item in value.items(): masked_item, item_masked = mask_pii_value(item) masked_dict[key] = masked_item @@ -51,8 +52,8 @@ def mask_pii_value(value: Any) -> Tuple[Any, bool]: # #region sanitize_imported_filter_for_assistant [C:2] [TYPE Function] # @BRIEF Build assistant-safe imported-filter payload with raw-value redaction metadata when sensitive tokens are detected. def sanitize_imported_filter_for_assistant( - filter_payload: Dict[str, Any], -) -> Dict[str, Any]: + filter_payload: dict[str, Any], +) -> dict[str, Any]: sanitized = deepcopy(filter_payload) masked_raw_value, was_masked = mask_pii_value(filter_payload.get("raw_value")) sanitized["raw_value"] = masked_raw_value @@ -67,7 +68,7 @@ def sanitize_imported_filter_for_assistant( # #region _mask_sensitive_text [C:2] [TYPE Function] # @BRIEF Redact likely PII substrings from one string while preserving non-sensitive delimiters and token shape. -def _mask_sensitive_text(value: str) -> Tuple[str, bool]: +def _mask_sensitive_text(value: str) -> tuple[str, bool]: masked = value masked_any = False diff --git a/backend/src/core/utils/superset_context_extractor/_recovery.py b/backend/src/core/utils/superset_context_extractor/_recovery.py index 94846a44..9b1c7365 100644 --- a/backend/src/core/utils/superset_context_extractor/_recovery.py +++ b/backend/src/core/utils/superset_context_extractor/_recovery.py @@ -6,11 +6,14 @@ # #endregion SupersetContextRecoveryMixin # #region _recovery_imports [TYPE Block] from __future__ import annotations + import json from copy import deepcopy -from typing import Any, Dict, List, Set, cast +from typing import Any, cast + from ...logger import belief_scope, logger from ._base import SupersetParsedContext + logger = cast(Any, logger) # #endregion _recovery_imports # #region SupersetContextRecoveryMixin [C:4] [TYPE Class] @@ -25,13 +28,13 @@ class SupersetContextRecoveryMixin: # @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]] def recover_imported_filters( self, parsed_context: SupersetParsedContext - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: with belief_scope("SupersetContextExtractor.recover_imported_filters"): - recovered_filters: List[Dict[str, Any]] = [] - seen_filter_keys: Set[str] = set() - metadata_filters: List[Dict[str, Any]] = [] - metadata_filters_by_id: Dict[str, Dict[str, Any]] = {} - def merge_recovered_filter(candidate: Dict[str, Any]) -> None: + recovered_filters: list[dict[str, Any]] = [] + seen_filter_keys: set[str] = set() + metadata_filters: list[dict[str, Any]] = [] + metadata_filters_by_id: dict[str, dict[str, Any]] = {} + def merge_recovered_filter(candidate: dict[str, Any]) -> None: filter_key = candidate["filter_name"].strip().lower() existing_index = next( ( @@ -242,10 +245,10 @@ class SupersetContextRecoveryMixin: # @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state. def _normalize_imported_filter_payload( self, - payload: Dict[str, Any], + payload: dict[str, Any], default_source: str, default_note: str, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: raw_value = payload.get("raw_value") if "raw_value" not in payload and "value" in payload: raw_value = payload.get("value") @@ -279,4 +282,4 @@ class SupersetContextRecoveryMixin: "notes": str(payload.get("notes") or default_note), } # #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload -# #endregion SupersetContextRecoveryMixin \ No newline at end of file +# #endregion SupersetContextRecoveryMixin diff --git a/backend/src/core/utils/superset_context_extractor/_templates.py b/backend/src/core/utils/superset_context_extractor/_templates.py index c44f7247..a06d9789 100644 --- a/backend/src/core/utils/superset_context_extractor/_templates.py +++ b/backend/src/core/utils/superset_context_extractor/_templates.py @@ -6,9 +6,12 @@ # #endregion SupersetContextTemplatesMixin # #region _templates_imports [TYPE Block] from __future__ import annotations + import re -from typing import Any, Dict, List, Optional, Set, cast +from typing import Any, cast + from ...logger import belief_scope, logger + logger = cast(Any, logger) # #endregion _templates_imports # #region SupersetContextTemplatesMixin [C:3] [TYPE Class] @@ -21,11 +24,11 @@ class SupersetContextTemplatesMixin: # @SIDE_EFFECT: none. # @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]] def discover_template_variables( - self, dataset_payload: Dict[str, Any] - ) -> List[Dict[str, Any]]: + self, dataset_payload: dict[str, Any] + ) -> list[dict[str, Any]]: with belief_scope("SupersetContextExtractor.discover_template_variables"): - discovered: List[Dict[str, Any]] = [] - seen_variable_names: Set[str] = set() + discovered: list[dict[str, Any]] = [] + seen_variable_names: set[str] = set() for expression_source in self._collect_query_bearing_expressions( dataset_payload ): @@ -111,9 +114,9 @@ class SupersetContextTemplatesMixin: # @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. # @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables] def _collect_query_bearing_expressions( - self, dataset_payload: Dict[str, Any] - ) -> List[str]: - expressions: List[str] = [] + self, dataset_payload: dict[str, Any] + ) -> list[str]: + expressions: list[str] = [] def append_expression(candidate: Any) -> None: if not isinstance(candidate, str): return @@ -147,8 +150,8 @@ class SupersetContextTemplatesMixin: # @PURPOSE: Append one deduplicated template-variable descriptor. def _append_template_variable( self, - discovered: List[Dict[str, Any]], - seen_variable_names: Set[str], + discovered: list[dict[str, Any]], + seen_variable_names: set[str], variable_name: str, expression_source: str, variable_kind: str, @@ -175,7 +178,7 @@ class SupersetContextTemplatesMixin: # #endregion SupersetContextTemplatesMixin._append_template_variable # #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [TYPE Function] [C:2] # @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it. - def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]: + def _extract_primary_jinja_identifier(self, expression: str) -> str | None: matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip()) if matched is None: return None @@ -195,7 +198,7 @@ class SupersetContextTemplatesMixin: # #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier # #region SupersetContextTemplatesMixin._normalize_default_literal [TYPE Function] [C:2] # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values. - def _normalize_default_literal(self, literal: Optional[str]) -> Any: + def _normalize_default_literal(self, literal: str | None) -> Any: normalized_literal = str(literal or "").strip() if not normalized_literal: return None @@ -218,4 +221,4 @@ class SupersetContextTemplatesMixin: except ValueError: return normalized_literal # #endregion SupersetContextTemplatesMixin._normalize_default_literal -# #endregion SupersetContextTemplatesMixin \ No newline at end of file +# #endregion SupersetContextTemplatesMixin diff --git a/backend/src/core/ws_log_handler.py b/backend/src/core/ws_log_handler.py index 5e06321b..25ad9e63 100644 --- a/backend/src/core/ws_log_handler.py +++ b/backend/src/core/ws_log_handler.py @@ -3,9 +3,9 @@ # @RELATION DEPENDS_ON -> [LogEntry] # @RELATION COMPOSES -> [CotJsonFormatter] import logging -from typing import Dict, Any, List, Optional from collections import deque from datetime import datetime +from typing import Any from pydantic import BaseModel, Field @@ -15,7 +15,7 @@ class LogEntry(BaseModel): timestamp: datetime = Field(default_factory=datetime.utcnow) level: str message: str - context: Optional[Dict[str, Any]] = None + context: dict[str, Any] | None = None # #endregion LogEntry @@ -58,7 +58,7 @@ class WebSocketLogHandler(logging.Handler): # #region WebSocketLogHandler.get_recent_logs [C:2] [TYPE Function] [SEMANTICS log,retrieve,buffer] # @BRIEF Returns a list of recent log entries from the buffer. - def get_recent_logs(self) -> List[LogEntry]: + def get_recent_logs(self) -> list[LogEntry]: """ Returns a list of recent log entries from the buffer. """ diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index ec339f43..7b17a722 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -13,47 +13,46 @@ # @RELATION CALLS -> [init_db] from pathlib import Path -from typing import Optional + from fastapi import Depends, HTTPException, status from fastapi.security import OAuth2PasswordBearer from jose import JWTError -from .core.plugin_loader import PluginLoader -from .core.task_manager import TaskManager -from .core.config_manager import ConfigManager -from .core.scheduler import SchedulerService -from .services.resource_service import ResourceService -from .services.mapping_service import MappingService -from .services.clean_release.repositories import ( - CandidateRepository, - ArtifactRepository, - ManifestRepository, - PolicyRepository, - ComplianceRepository, - ReportRepository, - ApprovalRepository, - PublicationRepository, - AuditRepository, - CleanReleaseAuditLog, -) -from .services.clean_release.repository import CleanReleaseRepository -from .services.clean_release.facade import CleanReleaseFacade -from .services.reports.report_service import ReportsService -from .core.database import init_db, get_auth_db, get_db -from .core.logger import logger + from .core.auth.jwt import decode_token from .core.auth.repository import AuthRepository +from .core.config_manager import ConfigManager +from .core.database import get_auth_db, get_db, init_db +from .core.logger import logger +from .core.plugin_loader import PluginLoader +from .core.scheduler import SchedulerService +from .core.task_manager import TaskManager from .models.auth import User +from .services.clean_release.facade import CleanReleaseFacade +from .services.clean_release.repositories import ( + ApprovalRepository, + ArtifactRepository, + AuditRepository, + CandidateRepository, + ComplianceRepository, + ManifestRepository, + PolicyRepository, + PublicationRepository, + ReportRepository, +) +from .services.clean_release.repository import CleanReleaseRepository +from .services.mapping_service import MappingService +from .services.resource_service import ResourceService # Initialize singletons lazily to avoid import-time DB side effects during test collection. # Use absolute path relative to this file to ensure plugins are found regardless of CWD project_root = Path(__file__).parent.parent.parent config_path = project_root / "config.json" -config_manager: Optional[ConfigManager] = None -plugin_loader: Optional[PluginLoader] = None -task_manager: Optional[TaskManager] = None -scheduler_service: Optional[SchedulerService] = None -resource_service: Optional[ResourceService] = None +config_manager: ConfigManager | None = None +plugin_loader: PluginLoader | None = None +task_manager: TaskManager | None = None +scheduler_service: SchedulerService | None = None +resource_service: ResourceService | None = None # #region get_config_manager [C:1] [TYPE Function] diff --git a/backend/src/models/__tests__/test_clean_release.py b/backend/src/models/__tests__/test_clean_release.py index b498ec82..4c2e4d51 100644 --- a/backend/src/models/__tests__/test_clean_release.py +++ b/backend/src/models/__tests__/test_clean_release.py @@ -2,26 +2,29 @@ # @RELATION VERIFIES -> [CleanReleaseModels] # @BRIEF Contract testing for Clean Release models # #endregion TestCleanReleaseModels -import pytest from datetime import datetime -from pydantic import ValidationError + +import pytest + from src.models.clean_release import ( - ReleaseCandidate, - ReleaseCandidateStatus, - ProfileType, + CheckFinalStatus, + CheckStageName, + CheckStageResult, + CheckStageStatus, + ClassificationType, CleanProfilePolicy, + ComplianceCheckRun, + ComplianceReport, DistributionManifest, + ExecutionMode, ManifestItem, ManifestSummary, - ClassificationType, - ComplianceCheckRun, - CheckFinalStatus, - CheckStageResult, - CheckStageName, - CheckStageStatus, - ComplianceReport, - ExecutionMode, + ProfileType, + ReleaseCandidate, + ReleaseCandidateStatus, ) + + # @TEST_FIXTURE: valid_enterprise_candidate @pytest.fixture def valid_candidate_data(): @@ -204,4 +207,4 @@ def test_report_validation(): violations_count=2, blocking_violations_count=0, ) -# #endregion test_report_validation \ No newline at end of file +# #endregion test_report_validation diff --git a/backend/src/models/__tests__/test_models.py b/backend/src/models/__tests__/test_models.py index c1e68600..7d26d33b 100644 --- a/backend/src/models/__tests__/test_models.py +++ b/backend/src/models/__tests__/test_models.py @@ -4,10 +4,13 @@ # @RELATION VERIFIES -> [ModelsPackage] import sys from pathlib import Path + # Add src to path sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src")) from src.core.config_models import Environment from src.core.logger import belief_scope + + # #region test_environment_model [TYPE Function] # @RELATION BINDS_TO -> test_models # @BRIEF Tests that Environment model correctly stores values. @@ -26,4 +29,4 @@ def test_environment_model(): assert env.name == "test-env" assert env.url == "http://localhost:8088/api/v1" # #endregion test_environment_model -# #endregion test_models \ No newline at end of file +# #endregion test_models diff --git a/backend/src/models/__tests__/test_report_models.py b/backend/src/models/__tests__/test_report_models.py index 3c9493a4..0f44e64a 100644 --- a/backend/src/models/__tests__/test_report_models.py +++ b/backend/src/models/__tests__/test_report_models.py @@ -4,9 +4,13 @@ # @LAYER: Domain import sys from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest from datetime import datetime, timedelta + +import pytest + + class TestTaskType: """Tests for the TaskType enum.""" def test_enum_values(self): @@ -40,7 +44,7 @@ class TestErrorContext: class TestTaskReport: """Tests for TaskReport model and its validators.""" def _make_report(self, **overrides): - from src.models.report import TaskReport, TaskType, ReportStatus + from src.models.report import ReportStatus, TaskReport, TaskType defaults = { "report_id": "rpt-001", "task_id": "task-001", @@ -146,7 +150,7 @@ class TestReportCollection: assert col.total == 0 assert col.has_next is False def test_with_items(self): - from src.models.report import ReportCollection, ReportQuery, TaskReport, TaskType, ReportStatus + from src.models.report import ReportCollection, ReportQuery, ReportStatus, TaskReport, TaskType report = TaskReport( report_id="r1", task_id="t1", task_type=TaskType.BACKUP, status=ReportStatus.SUCCESS, updated_at=datetime.utcnow(), @@ -161,7 +165,7 @@ class TestReportCollection: class TestReportDetailView: """Tests for ReportDetailView model.""" def test_valid_creation(self): - from src.models.report import ReportDetailView, TaskReport, TaskType, ReportStatus + from src.models.report import ReportDetailView, ReportStatus, TaskReport, TaskType report = TaskReport( report_id="r1", task_id="t1", task_type=TaskType.BACKUP, status=ReportStatus.SUCCESS, updated_at=datetime.utcnow(), @@ -173,7 +177,7 @@ class TestReportDetailView: assert detail.diagnostics is None assert detail.next_actions == [] def test_with_all_fields(self): - from src.models.report import ReportDetailView, TaskReport, TaskType, ReportStatus + from src.models.report import ReportDetailView, ReportStatus, TaskReport, TaskType report = TaskReport( report_id="r1", task_id="t1", task_type=TaskType.MIGRATION, status=ReportStatus.FAILED, updated_at=datetime.utcnow(), @@ -188,4 +192,4 @@ class TestReportDetailView: assert len(detail.timeline) == 1 assert detail.diagnostics["cause"] == "timeout" assert "Retry" in detail.next_actions -# #endregion test_report_models \ No newline at end of file +# #endregion test_report_models diff --git a/backend/src/models/assistant.py b/backend/src/models/assistant.py index b67db106..ad59e78e 100644 --- a/backend/src/models/assistant.py +++ b/backend/src/models/assistant.py @@ -6,7 +6,7 @@ from datetime import datetime -from sqlalchemy import Column, String, DateTime, JSON, Text +from sqlalchemy import JSON, Column, DateTime, String, Text from .mapping import Base diff --git a/backend/src/models/auth.py b/backend/src/models/auth.py index c3cb4e91..9dd550f0 100644 --- a/backend/src/models/auth.py +++ b/backend/src/models/auth.py @@ -7,8 +7,10 @@ import uuid from datetime import datetime -from sqlalchemy import Column, String, Boolean, DateTime, ForeignKey, Table + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String, Table from sqlalchemy.orm import relationship + from .mapping import Base diff --git a/backend/src/models/clean_release.py b/backend/src/models/clean_release.py index 0b686259..b2abd66f 100644 --- a/backend/src/models/clean_release.py +++ b/backend/src/models/clean_release.py @@ -9,19 +9,17 @@ # @INVARIANT: Immutable snapshots are never mutated; forbidden lifecycle transitions are rejected. from datetime import datetime -from dataclasses import dataclass from enum import Enum -from typing import List, Optional, Dict, Any -from pydantic import ConfigDict, Field, model_validator +from typing import Any + +from pydantic import ConfigDict, model_validator from pydantic.dataclasses import dataclass as pydantic_dataclass -from sqlalchemy import Column, String, DateTime, JSON, ForeignKey, Integer, Boolean -from sqlalchemy.orm import relationship -from .mapping import Base -from ..services.clean_release.enums import ( - CandidateStatus, RunStatus, ComplianceDecision, - ApprovalDecisionType, PublicationStatus, ClassificationType -) +from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String + +from ..services.clean_release.enums import CandidateStatus, ClassificationType, ComplianceDecision, PublicationStatus, RunStatus from ..services.clean_release.exceptions import IllegalTransitionError +from .mapping import Base + # #region ExecutionMode [TYPE Class] # @BRIEF Backward-compatible execution mode enum for legacy TUI/orchestrator tests. @@ -114,14 +112,14 @@ class ResourceSourceEntry: class ResourceSourceRegistry: registry_id: str name: str - entries: List[ResourceSourceEntry] + entries: list[ResourceSourceEntry] updated_at: datetime updated_by: str status: str = "ACTIVE" immutable: bool = True - allowed_hosts: Optional[List[str]] = None - allowed_schemes: Optional[List[str]] = None - allowed_source_types: Optional[List[str]] = None + allowed_hosts: list[str] | None = None + allowed_schemes: list[str] | None = None + allowed_source_types: list[str] | None = None @model_validator(mode="after") def populate_legacy_allowlists(self): @@ -148,12 +146,12 @@ class CleanProfilePolicy: profile: ProfileType active: bool internal_source_registry_ref: str - prohibited_artifact_categories: List[str] + prohibited_artifact_categories: list[str] effective_from: datetime - required_system_categories: Optional[List[str]] = None + required_system_categories: list[str] | None = None external_source_forbidden: bool = True immutable: bool = True - content_json: Optional[Dict[str, Any]] = None + content_json: dict[str, Any] | None = None @model_validator(mode="after") def validate_enterprise_policy(self): @@ -191,8 +189,8 @@ class ComplianceCheckRun: triggered_by: str execution_mode: ExecutionMode final_status: CheckFinalStatus - checks: List[CheckStageResult] - finished_at: Optional[datetime] = None + checks: list[CheckStageResult] + finished_at: datetime | None = None @model_validator(mode="after") def validate_final_status_alignment(self): @@ -233,7 +231,7 @@ class ComplianceCheckRun: # @POST: status advances only through legal transitions. class ReleaseCandidate(Base): __tablename__ = "clean_release_candidates" - + id = Column(String, primary_key=True) name = Column(String, nullable=True) # Added back for backward compatibility with some legacy DTOs version = Column(String, nullable=False) @@ -251,9 +249,7 @@ class ReleaseCandidate(Base): status = kwargs.get("status") if status is None: kwargs["status"] = CandidateStatus.DRAFT.value - elif isinstance(status, ReleaseCandidateStatus): - kwargs["status"] = status.value - elif isinstance(status, CandidateStatus): + elif isinstance(status, ReleaseCandidateStatus) or isinstance(status, CandidateStatus): kwargs["status"] = status.value if not str(kwargs.get("id", "")).strip(): raise ValueError("candidate_id must be non-empty") @@ -274,8 +270,8 @@ class ReleaseCandidate(Base): CandidateStatus.MANIFEST_BUILT: [CandidateStatus.CHECK_PENDING], CandidateStatus.CHECK_PENDING: [CandidateStatus.CHECK_RUNNING], CandidateStatus.CHECK_RUNNING: [ - CandidateStatus.CHECK_PASSED, - CandidateStatus.CHECK_BLOCKED, + CandidateStatus.CHECK_PASSED, + CandidateStatus.CHECK_BLOCKED, CandidateStatus.CHECK_ERROR ], CandidateStatus.CHECK_PASSED: [CandidateStatus.APPROVED, CandidateStatus.CHECK_PENDING], @@ -295,7 +291,7 @@ class ReleaseCandidate(Base): # @BRIEF Represents one artifact associated with a release candidate. class CandidateArtifact(Base): __tablename__ = "clean_release_artifacts" - + id = Column(String, primary_key=True) candidate_id = Column(String, ForeignKey("clean_release_candidates.id"), nullable=False) path = Column(String, nullable=False) @@ -315,7 +311,7 @@ class ManifestItem: category: str classification: ClassificationType reason: str - checksum: Optional[str] = None + checksum: str | None = None # #endregion ManifestItem # #region ManifestSummary [TYPE Class] @@ -331,7 +327,7 @@ class ManifestSummary: # @INVARIANT: Immutable after creation. class DistributionManifest(Base): __tablename__ = "clean_release_manifests" - + id = Column(String, primary_key=True) candidate_id = Column(String, ForeignKey("clean_release_candidates.id"), nullable=False) manifest_version = Column(Integer, nullable=False) @@ -364,7 +360,7 @@ class DistributionManifest(Base): expected_count = int(summary.included_count) + int(summary.excluded_count) if expected_count != len(items): raise ValueError("manifest summary counts must match items size") - + # Ensure required DB fields have defaults if missing if "manifest_version" not in kwargs: kwargs["manifest_version"] = 1 @@ -372,7 +368,7 @@ class DistributionManifest(Base): kwargs["artifacts_digest"] = kwargs.get("manifest_digest", "pending") if "source_snapshot_ref" not in kwargs: kwargs["source_snapshot_ref"] = "pending" - + # Pack items and summary into content_json if provided if items is not None or summary is not None: content = dict(kwargs.get("content_json") or {}) @@ -418,7 +414,7 @@ class DistributionManifest(Base): # @BRIEF Immutable registry snapshot for allowed sources. class SourceRegistrySnapshot(Base): __tablename__ = "clean_release_registry_snapshots" - + id = Column(String, primary_key=True) registry_id = Column(String, nullable=False) registry_version = Column(String, nullable=False) @@ -433,7 +429,7 @@ class SourceRegistrySnapshot(Base): # @BRIEF Immutable policy snapshot used to evaluate a run. class CleanPolicySnapshot(Base): __tablename__ = "clean_release_policy_snapshots" - + id = Column(String, primary_key=True) policy_id = Column(String, nullable=False) policy_version = Column(String, nullable=False) @@ -447,7 +443,7 @@ class CleanPolicySnapshot(Base): # @BRIEF Operational record for one compliance execution. class ComplianceRun(Base): __tablename__ = "clean_release_compliance_runs" - + id = Column(String, primary_key=True) candidate_id = Column(String, ForeignKey("clean_release_candidates.id"), nullable=False) manifest_id = Column(String, ForeignKey("clean_release_manifests.id"), nullable=False) @@ -472,7 +468,7 @@ class ComplianceRun(Base): # @BRIEF Stage-level execution record inside a run. class ComplianceStageRun(Base): __tablename__ = "clean_release_compliance_stage_runs" - + id = Column(String, primary_key=True) run_id = Column(String, ForeignKey("clean_release_compliance_runs.id"), nullable=False) stage_name = Column(String, nullable=False) @@ -505,7 +501,7 @@ class ViolationCategory(str, Enum): # @BRIEF Violation produced by a stage. class ComplianceViolation(Base): __tablename__ = "clean_release_compliance_violations" - + id = Column(String, primary_key=True) run_id = Column(String, ForeignKey("clean_release_compliance_runs.id"), nullable=False) stage_name = Column(String, nullable=False) @@ -565,11 +561,11 @@ class ComplianceViolation(Base): self.stage_name = value.value if isinstance(value, ViolationCategory) else str(value) @property - def location(self) -> Optional[str]: + def location(self) -> str | None: return self.artifact_path @property - def remediation(self) -> Optional[str]: + def remediation(self) -> str | None: return (self.evidence_json or {}).get("remediation") @property @@ -582,7 +578,7 @@ class ComplianceViolation(Base): # @INVARIANT: Immutable after creation. class ComplianceReport(Base): __tablename__ = "clean_release_compliance_reports" - + id = Column(String, primary_key=True) run_id = Column(String, ForeignKey("clean_release_compliance_runs.id"), nullable=False) candidate_id = Column(String, ForeignKey("clean_release_candidates.id"), nullable=False) @@ -639,7 +635,7 @@ class ComplianceReport(Base): return (self.summary_json or {}).get("operator_summary", "") @property - def structured_payload_ref(self) -> Optional[str]: + def structured_payload_ref(self) -> str | None: return (self.summary_json or {}).get("structured_payload_ref") @property @@ -655,7 +651,7 @@ class ComplianceReport(Base): # @BRIEF Approval or rejection bound to a candidate and report. class ApprovalDecision(Base): __tablename__ = "clean_release_approval_decisions" - + id = Column(String, primary_key=True) candidate_id = Column(String, ForeignKey("clean_release_candidates.id"), nullable=False) report_id = Column(String, ForeignKey("clean_release_compliance_reports.id"), nullable=False) @@ -669,7 +665,7 @@ class ApprovalDecision(Base): # @BRIEF Publication or revocation record. class PublicationRecord(Base): __tablename__ = "clean_release_publication_records" - + id = Column(String, primary_key=True) candidate_id = Column(String, ForeignKey("clean_release_candidates.id"), nullable=False) report_id = Column(String, ForeignKey("clean_release_compliance_reports.id"), nullable=False) @@ -683,9 +679,11 @@ class PublicationRecord(Base): # #region CleanReleaseAuditLog [TYPE Class] # @BRIEF Represents a persistent audit log entry for clean release actions. import uuid + + class CleanReleaseAuditLog(Base): __tablename__ = "clean_release_audit_logs" - + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) candidate_id = Column(String, index=True, nullable=True) action = Column(String, nullable=False) # e.g. "TRANSITION", "APPROVE", "PUBLISH" @@ -694,4 +692,4 @@ class CleanReleaseAuditLog(Base): details_json = Column(JSON, default=dict) # #endregion CleanReleaseAuditLog -# #endregion CleanReleaseModels \ No newline at end of file +# #endregion CleanReleaseModels diff --git a/backend/src/models/config.py b/backend/src/models/config.py index 47ff89cd..017c6228 100644 --- a/backend/src/models/config.py +++ b/backend/src/models/config.py @@ -6,7 +6,7 @@ # @RELATION DEPENDS_ON -> [MappingModels:Base] # @INVARIANT: Configuration payload and notification credentials must remain persisted as non-null JSON documents. -from sqlalchemy import Column, String, DateTime, JSON, Boolean +from sqlalchemy import JSON, Boolean, Column, DateTime, String from sqlalchemy.sql import func from .mapping import Base diff --git a/backend/src/models/connection.py b/backend/src/models/connection.py index ba3b2d6e..dad3149c 100644 --- a/backend/src/models/connection.py +++ b/backend/src/models/connection.py @@ -6,11 +6,14 @@ # # @INVARIANT: All primary keys are UUID strings. -from sqlalchemy import Column, String, Integer, DateTime -from sqlalchemy.sql import func -from .mapping import Base import uuid +from sqlalchemy import Column, DateTime, Integer, String +from sqlalchemy.sql import func + +from .mapping import Base + + # #region ConnectionConfig [C:1] [TYPE Class] # @BRIEF Stores credentials for external databases used for column mapping. class ConnectionConfig(Base): @@ -28,4 +31,4 @@ class ConnectionConfig(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) # #endregion ConnectionConfig -# #endregion ConnectionModels \ No newline at end of file +# #endregion ConnectionModels diff --git a/backend/src/models/dashboard.py b/backend/src/models/dashboard.py index 55890093..63f9b6ae 100644 --- a/backend/src/models/dashboard.py +++ b/backend/src/models/dashboard.py @@ -3,8 +3,9 @@ # @LAYER: Model # @RELATION USED_BY -> MigrationApi + from pydantic import BaseModel -from typing import List + # #region DashboardMetadata [C:1] [TYPE Class] # @BRIEF Represents a dashboard available for migration. @@ -18,11 +19,11 @@ class DashboardMetadata(BaseModel): # #region DashboardSelection [C:1] [TYPE Class] # @BRIEF Represents the user's selection of dashboards to migrate. class DashboardSelection(BaseModel): - selected_ids: List[int] + selected_ids: list[int] source_env_id: str target_env_id: str replace_db_config: bool = False fix_cross_filters: bool = True # #endregion DashboardSelection -# #endregion DashboardModels \ No newline at end of file +# #endregion DashboardModels diff --git a/backend/src/models/dataset_review.py b/backend/src/models/dataset_review.py index 614e1fe5..155b7e0b 100644 --- a/backend/src/models/dataset_review.py +++ b/backend/src/models/dataset_review.py @@ -14,66 +14,66 @@ # @RATIONALE: Original 984-line monolith violated INV_7 (400-line module limit). Decomposed into domain-focused sub-modules while preserving backward-compatible import paths. # @REJECTED: Keeping all models in a single file because it exceeded the fractal limit by 2.5x and accumulated structural erosion risk. +from src.models.dataset_review_pkg._clarification_models import ( # noqa: F401 + ClarificationAnswer, + ClarificationOption, + ClarificationQuestion, + ClarificationSession, +) from src.models.dataset_review_pkg._enums import ( # noqa: F401 - SessionStatus, - SessionPhase, - ReadinessState, - RecommendedAction, - SessionCollaboratorRole, + AnswerKind, + ApprovalState, + ArtifactFormat, + ArtifactType, BusinessSummarySource, - ConfidenceState, - FindingArea, - FindingSeverity, - ResolutionState, - SemanticSourceType, - TrustLevel, - SemanticSourceStatus, - FieldKind, - FieldProvenance, CandidateMatchType, CandidateStatus, - FilterSource, + ClarificationStatus, + ConfidenceState, + FieldKind, + FieldProvenance, FilterConfidenceState, FilterRecoveryStatus, - VariableKind, - MappingStatus, - MappingMethod, - MappingWarningLevel, - ApprovalState, - ClarificationStatus, - QuestionState, - AnswerKind, - PreviewStatus, + FilterSource, + FindingArea, + FindingSeverity, LaunchStatus, - ArtifactType, - ArtifactFormat, + MappingMethod, + MappingStatus, + MappingWarningLevel, + PreviewStatus, + QuestionState, + ReadinessState, + RecommendedAction, + ResolutionState, + SemanticSourceStatus, + SemanticSourceType, + SessionCollaboratorRole, + SessionPhase, + SessionStatus, + TrustLevel, + VariableKind, ) -from src.models.dataset_review_pkg._session_models import ( # noqa: F401 - SessionCollaborator, - DatasetReviewSession, -) -from src.models.dataset_review_pkg._profile_models import DatasetProfile # noqa: F401 -from src.models.dataset_review_pkg._finding_models import ValidationFinding # noqa: F401 -from src.models.dataset_review_pkg._semantic_models import ( # noqa: F401 - SemanticSource, - SemanticFieldEntry, - SemanticCandidate, +from src.models.dataset_review_pkg._execution_models import ( # noqa: F401 + CompiledPreview, + DatasetRunContext, + ExportArtifact, + SessionEvent, ) from src.models.dataset_review_pkg._filter_models import ( # noqa: F401 ImportedFilter, TemplateVariable, ) +from src.models.dataset_review_pkg._finding_models import ValidationFinding # noqa: F401 from src.models.dataset_review_pkg._mapping_models import ExecutionMapping # noqa: F401 -from src.models.dataset_review_pkg._clarification_models import ( # noqa: F401 - ClarificationSession, - ClarificationQuestion, - ClarificationOption, - ClarificationAnswer, +from src.models.dataset_review_pkg._profile_models import DatasetProfile # noqa: F401 +from src.models.dataset_review_pkg._semantic_models import ( # noqa: F401 + SemanticCandidate, + SemanticFieldEntry, + SemanticSource, ) -from src.models.dataset_review_pkg._execution_models import ( # noqa: F401 - CompiledPreview, - DatasetRunContext, - SessionEvent, - ExportArtifact, +from src.models.dataset_review_pkg._session_models import ( # noqa: F401 + DatasetReviewSession, + SessionCollaborator, ) # #endregion DatasetReviewModels diff --git a/backend/src/models/dataset_review_pkg/__init__.py b/backend/src/models/dataset_review_pkg/__init__.py index acb439d1..c62a887d 100644 --- a/backend/src/models/dataset_review_pkg/__init__.py +++ b/backend/src/models/dataset_review_pkg/__init__.py @@ -2,119 +2,119 @@ # @BRIEF Re-export all dataset review domain models from decomposed sub-modules for backward-compatible imports. # @LAYER: Domain +from src.models.dataset_review_pkg._clarification_models import ( + ClarificationAnswer, + ClarificationOption, + ClarificationQuestion, + ClarificationSession, +) from src.models.dataset_review_pkg._enums import ( - SessionStatus, - SessionPhase, - ReadinessState, - RecommendedAction, - SessionCollaboratorRole, + AnswerKind, + ApprovalState, + ArtifactFormat, + ArtifactType, BusinessSummarySource, - ConfidenceState, - FindingArea, - FindingSeverity, - ResolutionState, - SemanticSourceType, - TrustLevel, - SemanticSourceStatus, - FieldKind, - FieldProvenance, CandidateMatchType, CandidateStatus, - FilterSource, + ClarificationStatus, + ConfidenceState, + FieldKind, + FieldProvenance, FilterConfidenceState, FilterRecoveryStatus, - VariableKind, - MappingStatus, - MappingMethod, - MappingWarningLevel, - ApprovalState, - ClarificationStatus, - QuestionState, - AnswerKind, - PreviewStatus, + FilterSource, + FindingArea, + FindingSeverity, LaunchStatus, - ArtifactType, - ArtifactFormat, + MappingMethod, + MappingStatus, + MappingWarningLevel, + PreviewStatus, + QuestionState, + ReadinessState, + RecommendedAction, + ResolutionState, + SemanticSourceStatus, + SemanticSourceType, + SessionCollaboratorRole, + SessionPhase, + SessionStatus, + TrustLevel, + VariableKind, ) -from src.models.dataset_review_pkg._session_models import ( - SessionCollaborator, - DatasetReviewSession, -) -from src.models.dataset_review_pkg._profile_models import DatasetProfile -from src.models.dataset_review_pkg._finding_models import ValidationFinding -from src.models.dataset_review_pkg._semantic_models import ( - SemanticSource, - SemanticFieldEntry, - SemanticCandidate, +from src.models.dataset_review_pkg._execution_models import ( + CompiledPreview, + DatasetRunContext, + ExportArtifact, + SessionEvent, ) from src.models.dataset_review_pkg._filter_models import ( ImportedFilter, TemplateVariable, ) +from src.models.dataset_review_pkg._finding_models import ValidationFinding from src.models.dataset_review_pkg._mapping_models import ExecutionMapping -from src.models.dataset_review_pkg._clarification_models import ( - ClarificationSession, - ClarificationQuestion, - ClarificationOption, - ClarificationAnswer, +from src.models.dataset_review_pkg._profile_models import DatasetProfile +from src.models.dataset_review_pkg._semantic_models import ( + SemanticCandidate, + SemanticFieldEntry, + SemanticSource, ) -from src.models.dataset_review_pkg._execution_models import ( - CompiledPreview, - DatasetRunContext, - SessionEvent, - ExportArtifact, +from src.models.dataset_review_pkg._session_models import ( + DatasetReviewSession, + SessionCollaborator, ) __all__ = [ - "SessionStatus", - "SessionPhase", - "ReadinessState", - "RecommendedAction", - "SessionCollaboratorRole", + "AnswerKind", + "ApprovalState", + "ArtifactFormat", + "ArtifactType", "BusinessSummarySource", - "ConfidenceState", - "FindingArea", - "FindingSeverity", - "ResolutionState", - "SemanticSourceType", - "TrustLevel", - "SemanticSourceStatus", - "FieldKind", - "FieldProvenance", "CandidateMatchType", "CandidateStatus", - "FilterSource", + "ClarificationAnswer", + "ClarificationOption", + "ClarificationQuestion", + "ClarificationSession", + "ClarificationStatus", + "CompiledPreview", + "ConfidenceState", + "DatasetProfile", + "DatasetReviewSession", + "DatasetRunContext", + "ExecutionMapping", + "ExportArtifact", + "FieldKind", + "FieldProvenance", "FilterConfidenceState", "FilterRecoveryStatus", - "VariableKind", - "MappingStatus", - "MappingMethod", - "MappingWarningLevel", - "ApprovalState", - "ClarificationStatus", - "QuestionState", - "AnswerKind", - "PreviewStatus", - "LaunchStatus", - "ArtifactType", - "ArtifactFormat", - "SessionCollaborator", - "DatasetReviewSession", - "DatasetProfile", - "ValidationFinding", - "SemanticSource", - "SemanticFieldEntry", - "SemanticCandidate", + "FilterSource", + "FindingArea", + "FindingSeverity", "ImportedFilter", - "TemplateVariable", - "ExecutionMapping", - "ClarificationSession", - "ClarificationQuestion", - "ClarificationOption", - "ClarificationAnswer", - "CompiledPreview", - "DatasetRunContext", + "LaunchStatus", + "MappingMethod", + "MappingStatus", + "MappingWarningLevel", + "PreviewStatus", + "QuestionState", + "ReadinessState", + "RecommendedAction", + "ResolutionState", + "SemanticCandidate", + "SemanticFieldEntry", + "SemanticSource", + "SemanticSourceStatus", + "SemanticSourceType", + "SessionCollaborator", + "SessionCollaboratorRole", "SessionEvent", - "ExportArtifact", + "SessionPhase", + "SessionStatus", + "TemplateVariable", + "TrustLevel", + "ValidationFinding", + "VariableKind", ] # #endregion DatasetReviewModels diff --git a/backend/src/models/dataset_review_pkg/_clarification_models.py b/backend/src/models/dataset_review_pkg/_clarification_models.py index f0612d45..e68d419d 100644 --- a/backend/src/models/dataset_review_pkg/_clarification_models.py +++ b/backend/src/models/dataset_review_pkg/_clarification_models.py @@ -11,21 +11,23 @@ from datetime import datetime from sqlalchemy import ( Boolean, Column, - String, - Integer, - Text, DateTime, ForeignKey, + Integer, + String, + Text, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( + AnswerKind, ClarificationStatus, QuestionState, - AnswerKind, ) +from src.models.mapping import Base # #region ClarificationSession [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_execution_models.py b/backend/src/models/dataset_review_pkg/_execution_models.py index 59654694..03e491fd 100644 --- a/backend/src/models/dataset_review_pkg/_execution_models.py +++ b/backend/src/models/dataset_review_pkg/_execution_models.py @@ -8,23 +8,25 @@ import uuid from datetime import datetime from sqlalchemy import ( + JSON, Column, - String, - Text, DateTime, ForeignKey, - JSON, + String, + Text, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( - PreviewStatus, - LaunchStatus, - ArtifactType, ArtifactFormat, + ArtifactType, + LaunchStatus, + PreviewStatus, ) +from src.models.mapping import Base # #region CompiledPreview [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_filter_models.py b/backend/src/models/dataset_review_pkg/_filter_models.py index e1280b7c..a1b4fa21 100644 --- a/backend/src/models/dataset_review_pkg/_filter_models.py +++ b/backend/src/models/dataset_review_pkg/_filter_models.py @@ -8,25 +8,27 @@ import uuid from datetime import datetime from sqlalchemy import ( - Column, - String, - Text, + JSON, Boolean, + Column, DateTime, ForeignKey, - JSON, + String, + Text, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( - FilterSource, FilterConfidenceState, FilterRecoveryStatus, - VariableKind, + FilterSource, MappingStatus, + VariableKind, ) +from src.models.mapping import Base # #region ImportedFilter [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_finding_models.py b/backend/src/models/dataset_review_pkg/_finding_models.py index b5dcbc54..0861c5e9 100644 --- a/backend/src/models/dataset_review_pkg/_finding_models.py +++ b/backend/src/models/dataset_review_pkg/_finding_models.py @@ -9,20 +9,22 @@ from datetime import datetime from sqlalchemy import ( Column, - String, - Text, DateTime, ForeignKey, + String, + Text, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( FindingArea, FindingSeverity, ResolutionState, ) +from src.models.mapping import Base # #region ValidationFinding [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_mapping_models.py b/backend/src/models/dataset_review_pkg/_mapping_models.py index f9bdcaa4..9567f51c 100644 --- a/backend/src/models/dataset_review_pkg/_mapping_models.py +++ b/backend/src/models/dataset_review_pkg/_mapping_models.py @@ -8,23 +8,25 @@ import uuid from datetime import datetime from sqlalchemy import ( - Column, - String, - Text, + JSON, Boolean, + Column, DateTime, ForeignKey, - JSON, + String, + Text, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( + ApprovalState, MappingMethod, MappingWarningLevel, - ApprovalState, ) +from src.models.mapping import Base # #region ExecutionMapping [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_profile_models.py b/backend/src/models/dataset_review_pkg/_profile_models.py index 6592970f..850d0eeb 100644 --- a/backend/src/models/dataset_review_pkg/_profile_models.py +++ b/backend/src/models/dataset_review_pkg/_profile_models.py @@ -8,22 +8,24 @@ import uuid from datetime import datetime from sqlalchemy import ( + Boolean, Column, + DateTime, + Float, + ForeignKey, String, Text, - Float, - Boolean, - DateTime, - ForeignKey, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( BusinessSummarySource, ConfidenceState, ) +from src.models.mapping import Base # #region DatasetProfile [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_semantic_models.py b/backend/src/models/dataset_review_pkg/_semantic_models.py index 5d8e846a..ed16fc51 100644 --- a/backend/src/models/dataset_review_pkg/_semantic_models.py +++ b/backend/src/models/dataset_review_pkg/_semantic_models.py @@ -9,28 +9,30 @@ import uuid from datetime import datetime from sqlalchemy import ( - Column, - String, - Integer, - Text, - Float, Boolean, + Column, DateTime, + Float, ForeignKey, + Integer, + String, + Text, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( - SemanticSourceType, - TrustLevel, - SemanticSourceStatus, - FieldKind, - FieldProvenance, CandidateMatchType, CandidateStatus, + FieldKind, + FieldProvenance, + SemanticSourceStatus, + SemanticSourceType, + TrustLevel, ) +from src.models.mapping import Base # #region SemanticSource [C:2] [TYPE Class] diff --git a/backend/src/models/dataset_review_pkg/_session_models.py b/backend/src/models/dataset_review_pkg/_session_models.py index 65bc45fc..f1e5e00e 100644 --- a/backend/src/models/dataset_review_pkg/_session_models.py +++ b/backend/src/models/dataset_review_pkg/_session_models.py @@ -10,22 +10,24 @@ from datetime import datetime from sqlalchemy import ( Column, - String, - Integer, DateTime, ForeignKey, + Integer, + String, +) +from sqlalchemy import ( Enum as SQLEnum, ) from sqlalchemy.orm import relationship -from src.models.mapping import Base from src.models.dataset_review_pkg._enums import ( - SessionStatus, - SessionPhase, ReadinessState, RecommendedAction, SessionCollaboratorRole, + SessionPhase, + SessionStatus, ) +from src.models.mapping import Base # #region SessionCollaborator [C:2] [TYPE Class] diff --git a/backend/src/models/filter_state.py b/backend/src/models/filter_state.py index b88c1bdf..70318a53 100644 --- a/backend/src/models/filter_state.py +++ b/backend/src/models/filter_state.py @@ -4,7 +4,8 @@ # @LAYER: Models # @RELATION DEPENDS_ON -> [pydantic] -from typing import Any, Dict, List, Optional +from typing import Any + from pydantic import BaseModel, ConfigDict, Field @@ -13,12 +14,12 @@ from pydantic import BaseModel, ConfigDict, Field # @DATA_CONTRACT: Input[extraFormData: Dict, filterState: Dict, ownState: Optional[Dict]] -> Model[FilterState] class FilterState(BaseModel): """Single native filter state with extraFormData, filterState, and ownState.""" - + model_config = ConfigDict(extra="allow") - - extraFormData: Dict[str, Any] = Field(default_factory=dict, description="Extra form data for the filter") - filterState: Dict[str, Any] = Field(default_factory=dict, description="Current filter state") - ownState: Dict[str, Any] = Field(default_factory=dict, description="Own state of the filter") + + extraFormData: dict[str, Any] = Field(default_factory=dict, description="Extra form data for the filter") + filterState: dict[str, Any] = Field(default_factory=dict, description="Current filter state") + ownState: dict[str, Any] = Field(default_factory=dict, description="Own state of the filter") # #endregion FilterState @@ -27,16 +28,16 @@ class FilterState(BaseModel): # @DATA_CONTRACT: Input[Dict[filter_id, FilterState]] -> Model[NativeFilterDataMask] class NativeFilterDataMask(BaseModel): """Container for all native filter states in a dashboard.""" - + model_config = ConfigDict(extra="allow") - - filters: Dict[str, Any] = Field(default_factory=dict, description="Map of filter ID to filter state data") - - def get_filter_ids(self) -> List[str]: + + filters: dict[str, Any] = Field(default_factory=dict, description="Map of filter ID to filter state data") + + def get_filter_ids(self) -> list[str]: """Return list of all filter IDs.""" return list(self.filters.keys()) - - def get_extra_form_data(self, filter_id: str) -> Dict[str, Any]: + + def get_extra_form_data(self, filter_id: str) -> dict[str, Any]: """Get extraFormData for a specific filter.""" filter_state = self.filters.get(filter_id) if filter_state: @@ -50,22 +51,22 @@ class NativeFilterDataMask(BaseModel): # @DATA_CONTRACT: Input[dataMask: Dict, metadata: Dict] -> Model[ParsedNativeFilters] class ParsedNativeFilters(BaseModel): """Result of extracting native filters from a Superset URL.""" - + model_config = ConfigDict(extra="allow") - - dataMask: Dict[str, Any] = Field(default_factory=dict, description="Extracted dataMask from filters") - filter_type: Optional[str] = Field(default=None, description="Type of filter: permalink, native_filters_key, or native_filters") - dashboard_id: Optional[str] = Field(default=None, description="Dashboard ID if available") - permalink_key: Optional[str] = Field(default=None, description="Permalink key if used") - filter_state_key: Optional[str] = Field(default=None, description="Filter state key if used") - active_tabs: List[str] = Field(default_factory=list, description="Active tabs in dashboard") - anchor: Optional[str] = Field(default=None, description="Anchor position in dashboard") - chart_states: Dict[str, Any] = Field(default_factory=dict, description="Chart states in dashboard") - + + dataMask: dict[str, Any] = Field(default_factory=dict, description="Extracted dataMask from filters") + filter_type: str | None = Field(default=None, description="Type of filter: permalink, native_filters_key, or native_filters") + dashboard_id: str | None = Field(default=None, description="Dashboard ID if available") + permalink_key: str | None = Field(default=None, description="Permalink key if used") + filter_state_key: str | None = Field(default=None, description="Filter state key if used") + active_tabs: list[str] = Field(default_factory=list, description="Active tabs in dashboard") + anchor: str | None = Field(default=None, description="Anchor position in dashboard") + chart_states: dict[str, Any] = Field(default_factory=dict, description="Chart states in dashboard") + def has_filters(self) -> bool: """Check if any filters were extracted.""" return bool(self.dataMask) - + def get_filter_count(self) -> int: """Get the number of filters extracted.""" return len(self.dataMask) @@ -77,15 +78,15 @@ class ParsedNativeFilters(BaseModel): # @DATA_CONTRACT: Input[url: str, dashboard_id: Optional, filter_type: Optional, filters: Dict] -> Model[DashboardURLFilterExtraction] class DashboardURLFilterExtraction(BaseModel): """Result of parsing a Superset dashboard URL to extract filter state.""" - + model_config = ConfigDict(extra="allow") - + url: str = Field(..., description="Original dashboard URL") - dashboard_id: Optional[str] = Field(default=None, description="Extracted dashboard ID") - filter_type: Optional[str] = Field(default=None, description="Type of filter found") + dashboard_id: str | None = Field(default=None, description="Extracted dashboard ID") + filter_type: str | None = Field(default=None, description="Type of filter found") filters: ParsedNativeFilters = Field(default_factory=ParsedNativeFilters, description="Extracted filter data") success: bool = Field(default=True, description="Whether extraction was successful") - error: Optional[str] = Field(default=None, description="Error message if extraction failed") + error: str | None = Field(default=None, description="Error message if extraction failed") # #endregion DashboardURLFilterExtraction @@ -94,19 +95,19 @@ class DashboardURLFilterExtraction(BaseModel): # @DATA_CONTRACT: Input[append_keys: List[str], override_keys: List[str]] -> Model[ExtraFormDataMerge] class ExtraFormDataMerge(BaseModel): """Configuration for merging extraFormData between original and new filter values.""" - + # Keys that should be appended (arrays, filters) - append_keys: List[str] = Field( + append_keys: list[str] = Field( default_factory=lambda: ["filters", "extras", "columns", "metrics"], description="Keys that should be merged by appending" ) # Keys that should be overridden (single values) - override_keys: List[str] = Field( + override_keys: list[str] = Field( default_factory=lambda: ["time_range", "time_grain_sqla", "time_column", "granularity"], description="Keys that should be overridden by new values" ) - - def merge(self, original: Dict[str, Any], new: Dict[str, Any]) -> Dict[str, Any]: + + def merge(self, original: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]: """ Merge two extraFormData dictionaries. @@ -115,11 +116,11 @@ class ExtraFormDataMerge(BaseModel): @return: Merged extraFormData dictionary """ result = {} - + # Start with original for key, value in original.items(): result[key] = value - + # Apply overrides and appends from new for key, new_value in new.items(): if key in self.override_keys: @@ -134,9 +135,9 @@ class ExtraFormDataMerge(BaseModel): result[key] = new_value else: result[key] = new_value - + return result # #endregion ExtraFormDataMerge -# #endregion FilterStateModels \ No newline at end of file +# #endregion FilterStateModels diff --git a/backend/src/models/git.py b/backend/src/models/git.py index 5e989eb7..24bdbdd7 100644 --- a/backend/src/models/git.py +++ b/backend/src/models/git.py @@ -1,11 +1,14 @@ # #region GitModels [TYPE Module] [SEMANTICS sqlalchemy, git, model, config, sync] import enum -from datetime import datetime -from sqlalchemy import Column, String, Integer, DateTime, Enum, ForeignKey, Boolean import uuid +from datetime import datetime + +from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, String + from src.core.database import Base + class GitProvider(str, enum.Enum): GITHUB = "GITHUB" GITLAB = "GITLAB" diff --git a/backend/src/models/llm.py b/backend/src/models/llm.py index b3134663..6ddc84b1 100644 --- a/backend/src/models/llm.py +++ b/backend/src/models/llm.py @@ -3,11 +3,14 @@ # @LAYER: Domain # @RELATION INHERITS_FROM -> MappingModels:Base -from sqlalchemy import Column, String, Boolean, DateTime, JSON, Text, Time, ForeignKey -from datetime import datetime import uuid +from datetime import datetime + +from sqlalchemy import JSON, Boolean, Column, DateTime, String, Text, Time + from .mapping import Base + def generate_uuid(): return str(uuid.uuid4()) @@ -35,7 +38,7 @@ class ValidationPolicy(Base): # @BRIEF SQLAlchemy model for LLM provider configuration. class LLMProvider(Base): __tablename__ = "llm_providers" - + id = Column(String, primary_key=True, default=generate_uuid) provider_type = Column(String, nullable=False) # openai, openrouter, kilo name = Column(String, nullable=False) @@ -50,7 +53,7 @@ class LLMProvider(Base): # @BRIEF SQLAlchemy model for dashboard validation history. class ValidationRecord(Base): __tablename__ = "llm_validation_results" - + id = Column(String, primary_key=True, default=generate_uuid) task_id = Column(String, nullable=True, index=True) # Reference to TaskRecord dashboard_id = Column(String, nullable=False, index=True) @@ -63,4 +66,4 @@ class ValidationRecord(Base): raw_response = Column(Text, nullable=True) # #endregion ValidationRecord -# #endregion LlmModels \ No newline at end of file +# #endregion LlmModels diff --git a/backend/src/models/mapping.py b/backend/src/models/mapping.py index 1b0adf76..4051affe 100644 --- a/backend/src/models/mapping.py +++ b/backend/src/models/mapping.py @@ -8,11 +8,13 @@ # @INVARIANT: All primary keys are UUID strings. # @CONSTRAINT: source_env_id and target_env_id must be valid environment IDs. -from sqlalchemy import Column, String, Boolean, DateTime, ForeignKey, Enum as SQLEnum +import enum +import uuid + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String +from sqlalchemy import Enum as SQLEnum from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import func -import uuid -import enum Base = declarative_base() diff --git a/backend/src/models/profile.py b/backend/src/models/profile.py index ab76ee95..c98467ca 100644 --- a/backend/src/models/profile.py +++ b/backend/src/models/profile.py @@ -10,8 +10,10 @@ import uuid from datetime import datetime -from sqlalchemy import Column, String, Boolean, DateTime, ForeignKey + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, String from sqlalchemy.orm import relationship + from .mapping import Base diff --git a/backend/src/models/report.py b/backend/src/models/report.py index e949d4d1..83638446 100644 --- a/backend/src/models/report.py +++ b/backend/src/models/report.py @@ -10,7 +10,7 @@ from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any from pydantic import BaseModel, Field, field_validator, model_validator @@ -63,9 +63,9 @@ class ReportStatus(str, Enum): # @TEST_EDGE: missing_message -> {"code": "ERR_504"} # @RELATION DEPENDS_ON -> ReportModels class ErrorContext(BaseModel): - code: Optional[str] = None + code: str | None = None message: str - next_actions: List[str] = Field(default_factory=list) + next_actions: list[str] = Field(default_factory=list) # #endregion ErrorContext @@ -110,13 +110,13 @@ class TaskReport(BaseModel): task_id: str task_type: TaskType status: ReportStatus - started_at: Optional[datetime] = None + started_at: datetime | None = None updated_at: datetime summary: str - details: Optional[Dict[str, Any]] = None - validation_record: Optional[Dict[str, Any]] = None # Extended for US2 - error_context: Optional[ErrorContext] = None - source_ref: Optional[Dict[str, Any]] = None + details: dict[str, Any] | None = None + validation_record: dict[str, Any] | None = None # Extended for US2 + error_context: ErrorContext | None = None + source_ref: dict[str, Any] | None = None @field_validator("report_id", "task_id", "summary") @classmethod @@ -155,11 +155,11 @@ class TaskReport(BaseModel): class ReportQuery(BaseModel): page: int = Field(default=1, ge=1) page_size: int = Field(default=20, ge=1, le=100) - task_types: List[TaskType] = Field(default_factory=list) - statuses: List[ReportStatus] = Field(default_factory=list) - time_from: Optional[datetime] = None - time_to: Optional[datetime] = None - search: Optional[str] = Field(default=None, max_length=200) + task_types: list[TaskType] = Field(default_factory=list) + statuses: list[ReportStatus] = Field(default_factory=list) + time_from: datetime | None = None + time_to: datetime | None = None + search: str | None = Field(default=None, max_length=200) sort_by: str = Field(default="updated_at") sort_order: str = Field(default="desc") @@ -203,7 +203,7 @@ class ReportQuery(BaseModel): # @TEST_EDGE: negative_total -> {"items": [], "total": -5, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}} # @RELATION DEPENDS_ON -> ReportModels class ReportCollection(BaseModel): - items: List[TaskReport] + items: list[TaskReport] total: int = Field(ge=0) page: int = Field(ge=1) page_size: int = Field(ge=1) @@ -228,9 +228,9 @@ class ReportCollection(BaseModel): # @RELATION DEPENDS_ON -> ReportModels class ReportDetailView(BaseModel): report: TaskReport - timeline: List[Dict[str, Any]] = Field(default_factory=list) - diagnostics: Optional[Dict[str, Any]] = None - next_actions: List[str] = Field(default_factory=list) + timeline: list[dict[str, Any]] = Field(default_factory=list) + diagnostics: dict[str, Any] | None = None + next_actions: list[str] = Field(default_factory=list) # #endregion ReportDetailView diff --git a/backend/src/models/storage.py b/backend/src/models/storage.py index 90508b31..30345b17 100644 --- a/backend/src/models/storage.py +++ b/backend/src/models/storage.py @@ -2,9 +2,10 @@ from datetime import datetime from enum import Enum -from typing import Optional + from pydantic import BaseModel, Field + # #region FileCategory [C:1] [TYPE Class] # @BRIEF Enumeration of supported file categories in the storage system. class FileCategory(str, Enum): @@ -31,7 +32,7 @@ class StoredFile(BaseModel): size: int = Field(..., ge=0, description="Size of the file in bytes.") created_at: datetime = Field(..., description="Creation timestamp.") category: FileCategory = Field(..., description="Category of the file.") - mime_type: Optional[str] = Field(None, description="MIME type of the file.") + mime_type: str | None = Field(None, description="MIME type of the file.") # #endregion StoredFile -# #endregion StorageModels \ No newline at end of file +# #endregion StorageModels diff --git a/backend/src/models/task.py b/backend/src/models/task.py index a570387b..35981ba2 100644 --- a/backend/src/models/task.py +++ b/backend/src/models/task.py @@ -6,11 +6,14 @@ # # @INVARIANT: All primary keys are UUID strings. -from sqlalchemy import Column, String, DateTime, JSON, ForeignKey, Text, Integer, Index -from sqlalchemy.sql import func -from .mapping import Base import uuid +from sqlalchemy import JSON, Column, DateTime, ForeignKey, Index, Integer, String, Text +from sqlalchemy.sql import func + +from .mapping import Base + + # #region TaskRecord [C:1] [TYPE Class] # @BRIEF Represents a persistent record of a task execution. class TaskRecord(Base): @@ -107,4 +110,4 @@ class TaskLogRecord(Base): ) # #endregion TaskLogRecord -# #endregion TaskModels \ No newline at end of file +# #endregion TaskModels diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index 9c8c3b31..d2d106a5 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -3,14 +3,11 @@ # @LAYER: Domain # @RELATION INHERITS_FROM -> MappingModels:Base -from sqlalchemy import ( - Column, String, Boolean, DateTime, JSON, Text, - ForeignKey, Integer, UniqueConstraint, Index -) -from sqlalchemy.dialects.postgresql import JSON as PG_JSON -from datetime import datetime, timezone import uuid -import hashlib +from datetime import UTC, datetime + +from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint + from .mapping import Base @@ -57,8 +54,8 @@ class TranslationJob(Base): target_database_id = Column(String, nullable=True, comment="Superset database ID for SQL Lab insert target") created_by = Column(String, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) - updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # #endregion TranslationJob @@ -86,7 +83,7 @@ class TranslationRun(Base): config_hash = Column(String, nullable=True, comment="Hash of translation configuration state") dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time") created_by = Column(String, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) # #endregion TranslationRun @@ -104,7 +101,7 @@ class TranslationBatch(Base): failed_records = Column(Integer, default=0) started_at = Column(DateTime, nullable=True) completed_at = Column(DateTime, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) # #endregion TranslationBatch @@ -127,7 +124,7 @@ class TranslationRecord(Base): token_count_input = Column(Integer, nullable=True) token_count_output = Column(Integer, nullable=True) translation_duration_ms = Column(Integer, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) __table_args__ = ( Index("ix_translation_records_run_status", "run_id", "status"), @@ -146,7 +143,7 @@ class TranslationEvent(Base): event_type = Column(String, nullable=False) # JOB_CREATED, JOB_STARTED, JOB_COMPLETED, JOB_FAILED, etc. event_data = Column(JSON, nullable=True) created_by = Column(String, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) # #endregion TranslationEvent @@ -160,7 +157,7 @@ class TranslationPreviewSession(Base): run_id = Column(String, ForeignKey("translation_runs.id"), nullable=True) status = Column(String, nullable=False, default="ACTIVE") # ACTIVE, APPLIED, DISCARDED, EXPIRED created_by = Column(String, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) expires_at = Column(DateTime, nullable=True) # #endregion TranslationPreviewSession @@ -180,7 +177,7 @@ class TranslationPreviewRecord(Base): source_data = Column(JSON, nullable=True, comment="Original source row key columns for upsert matching") status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED feedback = Column(Text, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) # #endregion TranslationPreviewRecord @@ -196,8 +193,8 @@ class TerminologyDictionary(Base): target_dialect = Column(String, nullable=False) is_active = Column(Boolean, default=True) created_by = Column(String, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) - updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # #endregion TerminologyDictionary @@ -215,8 +212,8 @@ class DictionaryEntry(Base): origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated") origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction") origin_user_id = Column(String, nullable=True, comment="User who submitted the correction") - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) - updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) __table_args__ = ( UniqueConstraint( @@ -241,8 +238,8 @@ class TranslationSchedule(Base): next_run_at = Column(DateTime, nullable=True) execution_mode = Column(String, nullable=False, default="full", comment="full, new_key_only") created_by = Column(String, nullable=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) - updated_at = Column(DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) # #endregion TranslationSchedule @@ -254,7 +251,7 @@ class TranslationJobDictionary(Base): id = Column(String, primary_key=True, default=generate_uuid) job_id = Column(String, ForeignKey("translation_jobs.id"), nullable=False, index=True) dictionary_id = Column(String, ForeignKey("terminology_dictionaries.id"), nullable=False, index=True) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) __table_args__ = ( UniqueConstraint( @@ -287,8 +284,8 @@ class MetricSnapshot(Base): p50_duration_ms = Column(Integer, nullable=True) p95_duration_ms = Column(Integer, nullable=True) p99_duration_ms = Column(Integer, nullable=True) - snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) - created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC)) + created_at = Column(DateTime, default=lambda: datetime.now(UTC)) __table_args__ = ( Index("ix_metric_snapshots_job_date", "job_id", "snapshot_date"), diff --git a/backend/src/plugins/backup.py b/backend/src/plugins/backup.py index 732dfc11..d0d5da45 100755 --- a/backend/src/plugins/backup.py +++ b/backend/src/plugins/backup.py @@ -6,24 +6,20 @@ # @RELATION DEPENDS_ON -> superset_tool.utils # @RELATION USES -> TaskContext -from typing import Dict, Any, Optional from pathlib import Path +from typing import Any + from requests.exceptions import RequestException +from ..core.logger import belief_scope +from ..core.logger import logger as app_logger from ..core.plugin_base import PluginBase -from ..core.logger import belief_scope, logger as app_logger from ..core.superset_client import SupersetClient -from ..core.utils.network import SupersetAPIError -from ..core.utils.fileio import ( - save_and_unpack_dashboard, - archive_exports, - sanitize_filename, - consolidate_archive_folders, - remove_empty_directories, - RetentionPolicy -) -from ..dependencies import get_config_manager from ..core.task_manager.context import TaskContext +from ..core.utils.fileio import RetentionPolicy, archive_exports, consolidate_archive_folders, remove_empty_directories, sanitize_filename, save_and_unpack_dashboard +from ..core.utils.network import SupersetAPIError +from ..dependencies import get_config_manager + # #region BackupPlugin [TYPE Class] # @BRIEF Implementation of the backup plugin logic. @@ -90,12 +86,12 @@ class BackupPlugin(PluginBase): # @PRE: Plugin instance exists. # @POST: Returns dictionary schema. # @RETURN: Dict[str, Any] - JSON schema. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("get_schema"): config_manager = get_config_manager() envs = [e.name for e in config_manager.get_environments()] config_manager.get_config().settings.storage.root_path - + return { "type": "object", "properties": { @@ -116,58 +112,58 @@ class BackupPlugin(PluginBase): # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: Target environment must be configured. params must be a dictionary. # @POST: All dashboards are exported and archived. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): + async def execute(self, params: dict[str, Any], context: TaskContext | None = None): with belief_scope("execute"): config_manager = get_config_manager() - + # Support both parameter names: environment_id (for task creation) and env (for direct calls) env_id = params.get("environment_id") or params.get("env") dashboard_ids = params.get("dashboard_ids") or params.get("dashboards") - + # Log the incoming parameters for debugging log = context.logger if context else app_logger log.info(f"Backup parameters received: env_id={env_id}, dashboard_ids={dashboard_ids}") - + # Resolve environment name if environment_id is provided if env_id: env_config = next((e for e in config_manager.get_environments() if e.id == env_id), None) if env_config: params["env"] = env_config.name - + env = params.get("env") if not env: raise KeyError("env") - + log.info(f"Backup started for environment: {env}, selected dashboards: {dashboard_ids}") storage_settings = config_manager.get_config().settings.storage # Use 'backups' subfolder within the storage root backup_path = Path(storage_settings.root_path) / "backups" - + # Use TaskContext logger if available, otherwise fall back to app_logger log = context.logger if context else app_logger - + # Create sub-loggers for different components superset_log = log.with_source("superset_api") if context else log storage_log = log.with_source("storage") if context else log - + log.info(f"Starting backup for environment: {env}") try: config_manager = get_config_manager() if not config_manager.has_environments(): raise ValueError("No Superset environments configured. Please add an environment in Settings.") - + env_config = config_manager.get_environment(env) if not env_config: raise ValueError(f"Environment '{env}' not found in configuration.") - + client = SupersetClient(env_config) - + # Get all dashboards all_dashboard_count, all_dashboard_meta = client.get_dashboards() superset_log.info(f"Found {all_dashboard_count} total dashboards in environment") - + # Filter dashboards if specific IDs are provided if dashboard_ids: dashboard_ids_int = [int(did) for did in dashboard_ids] @@ -228,7 +224,7 @@ class BackupPlugin(PluginBase): "path": str(dashboard_dir) }) - except (SupersetAPIError, RequestException, IOError, OSError) as db_error: + except (SupersetAPIError, RequestException, OSError) as db_error: log.error(f"Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}") failed_dashboards.append({ "id": dashboard_id, @@ -236,7 +232,7 @@ class BackupPlugin(PluginBase): "error": str(db_error) }) continue - + consolidate_archive_folders(backup_path / env.upper()) remove_empty_directories(str(backup_path / env.upper())) @@ -252,7 +248,7 @@ class BackupPlugin(PluginBase): "failures": failed_dashboards } - except (RequestException, IOError, KeyError) as e: + except (OSError, RequestException, KeyError) as e: log.error(f"Fatal error during backup for {env}: {e}") raise e # endregion execute diff --git a/backend/src/plugins/debug.py b/backend/src/plugins/debug.py index b9427c70..62227b48 100644 --- a/backend/src/plugins/debug.py +++ b/backend/src/plugins/debug.py @@ -4,12 +4,14 @@ # @RELATION Inherits from PluginBase. Uses SupersetClient from core. # @RELATION USES -> TaskContext -from typing import Dict, Any, Optional +from typing import Any + +from ..core.logger import belief_scope, logger from ..core.plugin_base import PluginBase from ..core.superset_client import SupersetClient -from ..core.logger import logger, belief_scope from ..core.task_manager.context import TaskContext + # #region DebugPlugin [TYPE Class] # @BRIEF Plugin for system diagnostics and debugging. class DebugPlugin(PluginBase): @@ -75,7 +77,7 @@ class DebugPlugin(PluginBase): # @PRE: Plugin instance exists. # @POST: Returns dictionary schema. # @RETURN: Dict[str, Any] - JSON schema. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("get_schema"): return { "type": "object", @@ -118,17 +120,17 @@ class DebugPlugin(PluginBase): # @PRE: action must be provided in params. # @POST: Debug action is executed and results returned. # @RETURN: Dict[str, Any] - Execution results. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]: + async def execute(self, params: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]: with belief_scope("execute"): action = params.get("action") - + # Use TaskContext logger if available, otherwise fall back to app logger log = context.logger if context else logger debug_log = log.with_source("debug") if context else log superset_log = log.with_source("superset_api") if context else log - + debug_log.info(f"Executing debug action: {action}") - + if action == "test-db-api": return await self._test_db_api(params, superset_log) elif action == "get-dataset-structure": @@ -145,17 +147,17 @@ class DebugPlugin(PluginBase): # @PARAM: params (Dict) - Plugin parameters. # @PARAM: log - Logger instance for superset_api source. # @RETURN: Dict - Comparison results. - async def _test_db_api(self, params: Dict[str, Any], log) -> Dict[str, Any]: + async def _test_db_api(self, params: dict[str, Any], log) -> dict[str, Any]: with belief_scope("_test_db_api"): source_env_name = params.get("source_env") target_env_name = params.get("target_env") - + if not source_env_name or not target_env_name: raise ValueError("source_env and target_env are required for test-db-api") from ..dependencies import get_config_manager config_manager = get_config_manager() - + results = {} for name in [source_env_name, target_env_name]: log.info(f"Testing database API for environment: {name}") @@ -163,7 +165,7 @@ class DebugPlugin(PluginBase): if not env_config: log.error(f"Environment '{name}' not found.") raise ValueError(f"Environment '{name}' not found.") - + client = SupersetClient(env_config) client.authenticate() count, dbs = client.get_databases() @@ -172,7 +174,7 @@ class DebugPlugin(PluginBase): "count": count, "databases": dbs } - + return results # endregion _test_db_api @@ -183,30 +185,30 @@ class DebugPlugin(PluginBase): # @PARAM: params (Dict) - Plugin parameters. # @PARAM: log - Logger instance for superset_api source. # @RETURN: Dict - Dataset structure. - async def _get_dataset_structure(self, params: Dict[str, Any], log) -> Dict[str, Any]: + async def _get_dataset_structure(self, params: dict[str, Any], log) -> dict[str, Any]: with belief_scope("_get_dataset_structure"): env_name = params.get("env") dataset_id = params.get("dataset_id") - + if not env_name or dataset_id is None: raise ValueError("env and dataset_id are required for get-dataset-structure") log.info(f"Fetching structure for dataset {dataset_id} in {env_name}") - + from ..dependencies import get_config_manager config_manager = get_config_manager() env_config = config_manager.get_environment(env_name) if not env_config: log.error(f"Environment '{env_name}' not found.") raise ValueError(f"Environment '{env_name}' not found.") - + client = SupersetClient(env_config) client.authenticate() - + dataset_response = client.get_dataset(dataset_id) log.debug(f"Retrieved dataset structure for {dataset_id}") return dataset_response.get('result') or {} # endregion _get_dataset_structure # #endregion DebugPlugin -# #endregion DebugPluginModule \ No newline at end of file +# #endregion DebugPluginModule diff --git a/backend/src/plugins/git/llm_extension.py b/backend/src/plugins/git/llm_extension.py index 868a2758..c61f8dde 100644 --- a/backend/src/plugins/git/llm_extension.py +++ b/backend/src/plugins/git/llm_extension.py @@ -3,11 +3,13 @@ # @LAYER: Domain # @RELATION DEPENDS_ON -> [LLMClient] -from typing import List + from tenacity import retry, stop_after_attempt, wait_exponential -from ..llm_analysis.service import LLMClient + from ...core.logger import belief_scope, logger from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt +from ..llm_analysis.service import LLMClient + # #region GitLLMExtension [TYPE Class] # @BRIEF Provides LLM capabilities to the Git plugin. @@ -28,7 +30,7 @@ class GitLLMExtension: async def suggest_commit_message( self, diff: str, - history: List[str], + history: list[str], prompt_template: str = DEFAULT_LLM_PROMPTS["git_commit_prompt"], ) -> str: with belief_scope("suggest_commit_message"): @@ -37,20 +39,20 @@ class GitLLMExtension: prompt_template, {"history": history_text, "diff": diff}, ) - + logger.debug(f"[suggest_commit_message] Calling LLM with model: {self.client.default_model}") response = await self.client.client.chat.completions.create( model=self.client.default_model, messages=[{"role": "user", "content": prompt}], temperature=0.7 ) - + logger.debug(f"[suggest_commit_message] LLM Response: {response}") - + if not response or not hasattr(response, 'choices') or not response.choices: error_info = getattr(response, 'error', 'No choices in response') logger.error(f"[suggest_commit_message] Invalid LLM response. Error info: {error_info}") - + # If it's a timeout/provider error, we might want to throw to trigger retry if decorated # but for now we return a safe fallback to avoid UI crash return "Update dashboard configurations (LLM generation failed)" diff --git a/backend/src/plugins/git_plugin.py b/backend/src/plugins/git_plugin.py index 8d2add4f..bf203d07 100644 --- a/backend/src/plugins/git_plugin.py +++ b/backend/src/plugins/git_plugin.py @@ -11,25 +11,26 @@ # @INVARIANT: Все операции с Git должны выполняться через GitService. # @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset. -import os import io +import os import shutil import zipfile from pathlib import Path -from typing import Dict, Any, Optional -from src.core.plugin_base import PluginBase -from src.services.git_service import GitService -from src.core.logger import logger as app_logger, belief_scope +from typing import Any + from src.core.config_manager import ConfigManager +from src.core.logger import belief_scope +from src.core.logger import logger as app_logger +from src.core.plugin_base import PluginBase from src.core.superset_client import SupersetClient from src.core.task_manager.context import TaskContext -from src.core.database import SessionLocal -from src.core.mapping_service import IdMappingService +from src.services.git_service import GitService + # #region GitPlugin [TYPE Class] # @BRIEF Реализация плагина Git Integration для управления версиями дашбордов. class GitPlugin(PluginBase): - + # region __init__ [TYPE Function] # @PURPOSE: Инициализирует плагин и его зависимости. # @PRE: config.json exists or shared config_manager is available. @@ -38,7 +39,7 @@ class GitPlugin(PluginBase): with belief_scope("GitPlugin.__init__"): app_logger.info("Initializing GitPlugin.") self.git_service = GitService() - + # Robust config path resolution: # 1. Try absolute path from src/dependencies.py style if possible # 2. Try relative paths based on common execution patterns @@ -114,7 +115,7 @@ class GitPlugin(PluginBase): # @PRE: GitPlugin is initialized. # @POST: Returns a JSON schema dictionary. # @RETURN: Dict[str, Any] - Схема параметров. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("GitPlugin.get_schema"): return { "type": "object", @@ -145,20 +146,20 @@ class GitPlugin(PluginBase): # @RETURN: Dict[str, Any] - Статус и сообщение. # @RELATION: CALLS -> self._handle_sync # @RELATION: CALLS -> self._handle_deploy - async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]: + async def execute(self, task_data: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]: with belief_scope("GitPlugin.execute"): operation = task_data.get("operation") dashboard_id = task_data.get("dashboard_id") - + # Use TaskContext logger if available, otherwise fall back to app_logger log = context.logger if context else app_logger - + # Create sub-loggers for different components git_log = log.with_source("git") if context else log superset_log = log.with_source("superset_api") if context else log - + log.info(f"Executing operation: {operation} for dashboard {dashboard_id}") - + if operation == "sync": source_env_id = task_data.get("source_env_id") result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log) @@ -170,7 +171,7 @@ class GitPlugin(PluginBase): else: log.error(f"Unknown operation: {operation}") raise ValueError(f"Unknown operation: {operation}") - + log.info(f"Operation {operation} completed.") return result # endregion execute @@ -185,7 +186,7 @@ class GitPlugin(PluginBase): # @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория. # @RELATION: CALLS -> src.services.git_service.GitService.get_repo # @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard - async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]: + async def _handle_sync(self, dashboard_id: int, source_env_id: str | None = None, log=None, git_log=None, superset_log=None) -> dict[str, str]: with belief_scope("GitPlugin._handle_sync"): try: # 1. Получение репозитория @@ -201,14 +202,14 @@ class GitPlugin(PluginBase): # 3. Экспорт дашборда superset_log.info(f"Exporting dashboard {dashboard_id} from {env.name}") zip_bytes, _ = client.export_dashboard(dashboard_id) - + # 4. Распаковка с выравниванием структуры (flattening) git_log.info(f"Unpacking export to {repo_path}") - + # Список папок/файлов, которые мы ожидаем от Superset managed_dirs = ["dashboards", "charts", "datasets", "databases"] managed_files = ["metadata.yaml"] - + # Очистка старых данных перед распаковкой, чтобы не оставалось "призраков" for d in managed_dirs: d_path = repo_path / d @@ -225,23 +226,23 @@ class GitPlugin(PluginBase): namelist = zf.namelist() if not namelist: raise ValueError("Export ZIP is empty") - + root_folder = namelist[0].split('/')[0] git_log.info(f"Detected root folder in ZIP: {root_folder}") - + for member in zf.infolist(): if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1: # Убираем префикс папки relative_path = member.filename[len(root_folder)+1:] target_path = repo_path / relative_path - + if member.is_dir(): target_path.mkdir(parents=True, exist_ok=True) else: target_path.parent.mkdir(parents=True, exist_ok=True) with zf.open(member) as source, open(target_path, "wb") as target: shutil.copyfileobj(source, target) - + # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff) try: repo.git.add(A=True) @@ -269,7 +270,7 @@ class GitPlugin(PluginBase): # @RETURN: Dict[str, Any] - Результат деплоя. # @SIDE_EFFECT: Создает и удаляет временный ZIP-файл. # @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard - async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]: + async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> dict[str, Any]: with belief_scope("GitPlugin._handle_deploy"): try: if not env_id: @@ -282,10 +283,10 @@ class GitPlugin(PluginBase): # 2. Упаковка в ZIP git_log.info(f"Packing repository {repo_path} for deployment.") zip_buffer = io.BytesIO() - + # Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/) root_dir_name = f"dashboard_export_{dashboard_id}" - + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf: for root, dirs, files in os.walk(repo_path): if ".git" in dirs: @@ -297,14 +298,14 @@ class GitPlugin(PluginBase): # Prepend the root directory name to the archive path arcname = Path(root_dir_name) / file_path.relative_to(repo_path) zf.write(file_path, arcname) - + zip_buffer.seek(0) - + # 3. Настройка клиента Superset env = self.config_manager.get_environment(env_id) if not env: raise ValueError(f"Environment {env_id} not found") - + client = SupersetClient(env) client.authenticate() @@ -313,7 +314,7 @@ class GitPlugin(PluginBase): git_log.info(f"Saving temporary zip to {temp_zip_path}") with open(temp_zip_path, "wb") as f: f.write(zip_buffer.getvalue()) - + try: app_logger.reason(f"Importing dashboard to {env.name}", extra={"src": "_handle_deploy"}) result = client.import_dashboard(temp_zip_path) @@ -334,21 +335,21 @@ class GitPlugin(PluginBase): # @PRE: env_id is a string or None. # @POST: Returns an Environment object from config or DB. # @RETURN: Environment - Объект конфигурации окружения. - def _get_env(self, env_id: Optional[str] = None): + def _get_env(self, env_id: str | None = None): with belief_scope("GitPlugin._get_env"): app_logger.reason(f"Fetching environment for ID: {env_id}", extra={"src": "_get_env"}) - + # Priority 1: ConfigManager (config.json) if env_id: env = self.config_manager.get_environment(env_id) if env: app_logger.reflect(f"Found environment by ID in ConfigManager: {env.name}", extra={"src": "_get_env"}) return env - + # Priority 2: Database (DeploymentEnvironment) from src.core.database import SessionLocal from src.models.git import DeploymentEnvironment - + db = SessionLocal() try: if env_id: @@ -373,7 +374,7 @@ class GitPlugin(PluginBase): ) finally: db.close() - + # Priority 3: ConfigManager Default (if no env_id provided) envs = self.config_manager.get_environments() if envs: @@ -384,15 +385,15 @@ class GitPlugin(PluginBase): if env: app_logger.reflect(f"Found environment {env_id} in ConfigManager list", extra={"src": "_get_env"}) return env - + if not env_id: app_logger.reflect(f"Using first environment from ConfigManager: {envs[0].name}", extra={"src": "_get_env"}) return envs[0] - + app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}") raise ValueError("No environments configured. Please add a Superset Environment in Settings.") # endregion _get_env # endregion initialize # #endregion GitPlugin -# #endregion GitPluginModule \ No newline at end of file +# #endregion GitPluginModule diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index 476bf02a..31c31b5b 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -5,11 +5,12 @@ # @RELATION DEPENDS_on -> pydantic # @RELATION DEPENDs_on -> pydantic -from typing import List, Optional -from pydantic import BaseModel, Field from datetime import datetime from enum import Enum +from pydantic import BaseModel, Field + + # #region LLMProviderType [TYPE Class] # @BRIEF Enum for supported LLM providers. class LLMProviderType(str, Enum): @@ -21,11 +22,11 @@ class LLMProviderType(str, Enum): # #region LLMProviderConfig [TYPE Class] # @BRIEF Configuration for an LLM provider. class LLMProviderConfig(BaseModel): - id: Optional[str] = None + id: str | None = None provider_type: LLMProviderType name: str base_url: str - api_key: Optional[str] = None + api_key: str | None = None default_model: str is_active: bool = True # #endregion LLMProviderConfig @@ -44,20 +45,20 @@ class ValidationStatus(str, Enum): class DetectedIssue(BaseModel): severity: ValidationStatus message: str - location: Optional[str] = None + location: str | None = None # #endregion DetectedIssue # #region ValidationResult [TYPE Class] # @BRIEF Model for dashboard validation result. class ValidationResult(BaseModel): - id: Optional[str] = None + id: str | None = None dashboard_id: str timestamp: datetime = Field(default_factory=datetime.utcnow) status: ValidationStatus - screenshot_path: Optional[str] = None - issues: List[DetectedIssue] + screenshot_path: str | None = None + issues: list[DetectedIssue] summary: str - raw_response: Optional[str] = None + raw_response: str | None = None # #endregion ValidationResult # #endregion LLMAnalysisModels diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index 06e11df9..68f4394a 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -8,32 +8,34 @@ # @RELATION USES -> TaskContext # @INVARIANT: All LLM interactions must be executed as asynchronous tasks. -from typing import Dict, Any, Optional -import os import json +import os from datetime import datetime, timedelta -from ...core.plugin_base import PluginBase -from ...core.logger import belief_scope, logger +from typing import Any + from ...core.database import SessionLocal -from ...services.llm_provider import LLMProviderService +from ...core.logger import belief_scope, logger +from ...core.plugin_base import PluginBase from ...core.superset_client import SupersetClient -from .service import ScreenshotService, LLMClient -from .models import LLMProviderType, ValidationStatus, ValidationResult, DetectedIssue -from ...models.llm import ValidationRecord, ValidationPolicy from ...core.task_manager.context import TaskContext -from ...services.notifications.service import NotificationService +from ...models.llm import ValidationPolicy, ValidationRecord from ...services.llm_prompt_templates import ( DEFAULT_LLM_PROMPTS, is_multimodal_model, normalize_llm_settings, render_prompt, ) +from ...services.llm_provider import LLMProviderService +from ...services.notifications.service import NotificationService +from .models import DetectedIssue, LLMProviderType, ValidationResult, ValidationStatus +from .service import LLMClient, ScreenshotService + # #region _is_masked_or_invalid_api_key [TYPE Function] # @BRIEF Guards against placeholder or malformed API keys in runtime. # @PRE: value may be None. # @POST: Returns True when value cannot be used for authenticated provider calls. -def _is_masked_or_invalid_api_key(value: Optional[str]) -> bool: +def _is_masked_or_invalid_api_key(value: str | None) -> bool: key = (value or "").strip() if not key: return True @@ -77,7 +79,7 @@ class DashboardValidationPlugin(PluginBase): def version(self) -> str: return "1.0.0" - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: return { "type": "object", "properties": { @@ -95,19 +97,19 @@ class DashboardValidationPlugin(PluginBase): # @PRE: params contains dashboard_id, environment_id, and provider_id. # @POST: Returns a dictionary with validation results and persists them to the database. # @SIDE_EFFECT: Captures a screenshot, calls LLM API, and writes to the database. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): + async def execute(self, params: dict[str, Any], context: TaskContext | None = None): with belief_scope("execute", f"plugin_id={self.id}"): validation_started_at = datetime.utcnow() # Use TaskContext logger if available, otherwise fall back to app logger log = context.logger if context else logger - + # Create sub-loggers for different components llm_log = log.with_source("llm") if context else log screenshot_log = log.with_source("screenshot") if context else log superset_log = log.with_source("superset_api") if context else log - + log.info(f"Executing {self.name} with params: {params}") - + dashboard_id_raw = params.get("dashboard_id") dashboard_id = str(dashboard_id_raw) if dashboard_id_raw is not None else None env_id = params.get("environment_id") @@ -129,7 +131,7 @@ class DashboardValidationPlugin(PluginBase): if not db_provider: log.error(f"LLM Provider {provider_id} not found") raise ValueError(f"LLM Provider {provider_id} not found") - + llm_log.debug("Retrieved provider config:") llm_log.debug(f" Provider ID: {db_provider.id}") llm_log.debug(f" Provider Name: {db_provider.name}") @@ -141,10 +143,10 @@ class DashboardValidationPlugin(PluginBase): raise ValueError( "Dashboard validation requires a multimodal model (image input support)." ) - + api_key = llm_service.get_decrypted_api_key(provider_id) llm_log.debug(f"API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...") - + # Check if API key was successfully decrypted if _is_masked_or_invalid_api_key(api_key): raise ValueError( @@ -154,14 +156,14 @@ class DashboardValidationPlugin(PluginBase): # 3. Capture Screenshot screenshot_service = ScreenshotService(env) - + storage_root = config_mgr.get_config().settings.storage.root_path screenshots_dir = os.path.join(storage_root, "screenshots") os.makedirs(screenshots_dir, exist_ok=True) - + filename = f"{dashboard_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png" screenshot_path = os.path.join(screenshots_dir, filename) - + screenshot_started_at = datetime.utcnow() screenshot_log.info(f"Capturing screenshot for dashboard {dashboard_id}") await screenshot_service.capture_dashboard(dashboard_id, screenshot_path) @@ -173,10 +175,10 @@ class DashboardValidationPlugin(PluginBase): logs_fetch_started_at = datetime.utcnow() try: client = SupersetClient(env) - + # Calculate time window (last 24 hours) start_time = (datetime.now() - timedelta(hours=24)).isoformat() - + # Construct filter for logs # Note: We filter by dashboard_id matching the object query_params = { @@ -189,28 +191,28 @@ class DashboardValidationPlugin(PluginBase): "page": 0, "page_size": 100 } - + superset_log.debug(f"Fetching logs for dashboard {dashboard_id}") response = client.network.request( method="GET", endpoint="/log/", params={"q": json.dumps(query_params)} ) - + if isinstance(response, dict) and "result" in response: for item in response["result"]: action = item.get("action", "unknown") dttm = item.get("dttm", "") details = item.get("json", "") logs.append(f"[{dttm}] {action}: {details}") - + if not logs: logs = ["No recent logs found for this dashboard."] superset_log.debug("No recent logs found for this dashboard") except Exception as e: superset_log.warning(f"Failed to fetch logs from environment: {e}") - logs = [f"Error fetching remote logs: {str(e)}"] + logs = [f"Error fetching remote logs: {e!s}"] logs_fetch_finished_at = datetime.utcnow() # 5. Analyze with LLM @@ -220,7 +222,7 @@ class DashboardValidationPlugin(PluginBase): base_url=db_provider.base_url, default_model=db_provider.default_model ) - + llm_log.info(f"Analyzing dashboard {dashboard_id} with LLM") llm_settings = normalize_llm_settings(config_mgr.get_config().settings.llm) dashboard_prompt = llm_settings["prompts"].get( @@ -234,7 +236,7 @@ class DashboardValidationPlugin(PluginBase): prompt_template=dashboard_prompt, ) llm_call_finished_at = datetime.utcnow() - + # Log analysis summary to task logs for better visibility llm_log.info(f"[ANALYSIS_SUMMARY] Status: {analysis['status']}") llm_log.info(f"[ANALYSIS_SUMMARY] Summary: {analysis['summary']}") @@ -300,7 +302,7 @@ class DashboardValidationPlugin(PluginBase): policy = None if policy_id: policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == policy_id).first() - + notification_service = NotificationService(db, config_mgr) await notification_service.dispatch_report( record=db_record, @@ -312,7 +314,7 @@ class DashboardValidationPlugin(PluginBase): # Final log to ensure all analysis is visible in task logs log.info(f"Validation completed for dashboard {dashboard_id}. Status: {validation_result.status.value}") - + return result_payload finally: @@ -340,7 +342,7 @@ class DocumentationPlugin(PluginBase): def version(self) -> str: return "1.0.0" - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: return { "type": "object", "properties": { @@ -358,17 +360,17 @@ class DocumentationPlugin(PluginBase): # @PRE: params contains dataset_id, environment_id, and provider_id. # @POST: Returns generated documentation and updates the dataset in Superset. # @SIDE_EFFECT: Calls LLM API and updates dataset metadata in Superset. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): + async def execute(self, params: dict[str, Any], context: TaskContext | None = None): with belief_scope("execute", f"plugin_id={self.id}"): # Use TaskContext logger if available, otherwise fall back to app logger log = context.logger if context else logger - + # Create sub-loggers for different components llm_log = log.with_source("llm") if context else log superset_log = log.with_source("superset_api") if context else log - + log.info(f"Executing {self.name} with params: {params}") - + dataset_id = params.get("dataset_id") env_id = params.get("environment_id") provider_id = params.get("provider_id") @@ -389,17 +391,17 @@ class DocumentationPlugin(PluginBase): if not db_provider: log.error(f"LLM Provider {provider_id} not found") raise ValueError(f"LLM Provider {provider_id} not found") - + llm_log.debug("Retrieved provider config:") llm_log.debug(f" Provider ID: {db_provider.id}") llm_log.debug(f" Provider Name: {db_provider.name}") llm_log.debug(f" Provider Type: {db_provider.provider_type}") llm_log.debug(f" Base URL: {db_provider.base_url}") llm_log.debug(f" Default Model: {db_provider.default_model}") - + api_key = llm_service.get_decrypted_api_key(provider_id) llm_log.debug(f"API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...") - + # Check if API key was successfully decrypted if _is_masked_or_invalid_api_key(api_key): raise ValueError( @@ -410,10 +412,10 @@ class DocumentationPlugin(PluginBase): # 3. Fetch Metadata (US2 / T024) from ...core.superset_client import SupersetClient client = SupersetClient(env) - + superset_log.debug(f"Fetching dataset {dataset_id}") dataset = client.get_dataset(int(dataset_id)) - + # Extract columns and existing descriptions columns_data = [] for col in dataset.get("columns", []): @@ -431,7 +433,7 @@ class DocumentationPlugin(PluginBase): base_url=db_provider.base_url, default_model=db_provider.default_model ) - + llm_settings = normalize_llm_settings(config_mgr.get_config().settings.llm) documentation_prompt = llm_settings["prompts"].get( "documentation_prompt", @@ -444,7 +446,7 @@ class DocumentationPlugin(PluginBase): "columns_json": json.dumps(columns_data, ensure_ascii=False), }, ) - + # Using a generic chat completion for text-only US2 llm_log.info(f"Generating documentation for dataset {dataset_id}") doc_result = await llm_client.get_json_completion([{"role": "user", "content": prompt}]) @@ -454,7 +456,7 @@ class DocumentationPlugin(PluginBase): "description": doc_result["dataset_description"], "columns": [] } - + # Map generated descriptions back to column IDs for col_doc in doc_result["column_descriptions"]: for col in dataset.get("columns", []): @@ -466,9 +468,9 @@ class DocumentationPlugin(PluginBase): superset_log.info(f"Updating dataset {dataset_id} with generated documentation") client.update_dataset(int(dataset_id), update_payload) - + log.info(f"Documentation completed for dataset {dataset_id}") - + return doc_result finally: diff --git a/backend/src/plugins/llm_analysis/scheduler.py b/backend/src/plugins/llm_analysis/scheduler.py index 924ff949..976fac51 100644 --- a/backend/src/plugins/llm_analysis/scheduler.py +++ b/backend/src/plugins/llm_analysis/scheduler.py @@ -3,20 +3,22 @@ # @LAYER: Domain # @RELATION DEPENDS_ON -> [SchedulerService] -from typing import Dict, Any -from ...dependencies import get_task_manager, get_scheduler_service +from typing import Any + from ...core.logger import belief_scope, logger +from ...dependencies import get_scheduler_service, get_task_manager + # #region schedule_dashboard_validation [TYPE Function] # @BRIEF Schedules a recurring dashboard validation task. # @SIDE_EFFECT: Adds a job to the scheduler service. -def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: Dict[str, Any]): +def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: dict[str, Any]): with belief_scope("schedule_dashboard_validation", f"dashboard_id={dashboard_id}"): scheduler = get_scheduler_service() task_manager = get_task_manager() - + job_id = f"llm_val_{dashboard_id}" - + async def job_func(): await task_manager.create_task( plugin_id="llm_dashboard_validation", @@ -38,7 +40,7 @@ def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, param # #region _parse_cron [TYPE Function] # @BRIEF Basic cron parser placeholder. -def _parse_cron(cron: str) -> Dict[str, str]: +def _parse_cron(cron: str) -> dict[str, str]: # Basic cron parser placeholder parts = cron.split() if len(parts) != 5: diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index ee2fe28e..71333392 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -8,20 +8,24 @@ import asyncio import base64 -import json import io +import json import os +from typing import Any from urllib.parse import urlsplit -from typing import List, Dict, Any + import httpx +from openai import AsyncOpenAI, RateLimitError +from openai import AuthenticationError as OpenAIAuthenticationError from PIL import Image from playwright.async_api import async_playwright -from openai import AsyncOpenAI, RateLimitError, AuthenticationError as OpenAIAuthenticationError -from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception -from .models import LLMProviderType -from ...core.logger import belief_scope, logger +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential + from ...core.config_models import Environment +from ...core.logger import belief_scope, logger from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt +from .models import LLMProviderType + # #region ScreenshotService [TYPE Class] # @BRIEF Handles capturing screenshots of Superset dashboards. @@ -54,7 +58,7 @@ class ScreenshotService: # @PURPOSE: Enumerate page and child frames where login controls may be rendered. # @PRE: page is a Playwright page-like object. # @POST: Returns ordered roots starting with main page followed by frames. - def _iter_login_roots(self, page) -> List[Any]: + def _iter_login_roots(self, page) -> list[Any]: roots = [page] page_frames = getattr(page, "frames", []) try: @@ -70,8 +74,8 @@ class ScreenshotService: # @PURPOSE: Collect hidden form fields required for direct login POST fallback. # @PRE: Login page is loaded. # @POST: Returns hidden input name/value mapping aggregated from page and child frames. - async def _extract_hidden_login_fields(self, page) -> Dict[str, str]: - hidden_fields: Dict[str, str] = {} + async def _extract_hidden_login_fields(self, page) -> dict[str, str]: + hidden_fields: dict[str, str] = {} for root in self._iter_login_roots(page): try: locator = root.locator("input[type='hidden'][name]") @@ -296,7 +300,7 @@ class ScreenshotService: base_ui_url = self.env.url.rstrip("/") if base_ui_url.endswith("/api/v1"): base_ui_url = base_ui_url[:-len("/api/v1")] - + # Create browser context with realistic headers context = await browser.new_context( viewport={'width': 1280, 'height': 720}, @@ -320,7 +324,7 @@ class ScreenshotService: # 1. Navigate to login page and authenticate login_url = f"{base_ui_url.rstrip('/')}/login/" logger.info(f"[DEBUG] Navigating to login page: {login_url}") - + response = await self._goto_resilient( page, login_url, @@ -330,10 +334,10 @@ class ScreenshotService: ) if response: logger.info(f"[DEBUG] Login page response status: {response.status}") - + # Wait for login form to be ready await page.wait_for_load_state("domcontentloaded") - + # More exhaustive list of selectors for various Superset versions/themes selectors = { "username": ['input[name="username"]', 'input#username', 'input[placeholder*="Username"]', 'input[type="text"]'], @@ -341,7 +345,7 @@ class ScreenshotService: "submit": ['button[type="submit"]', 'button#submit', '.btn-primary', 'input[type="submit"]'] } logger.info("[DEBUG] Attempting to find login form elements...") - + try: used_direct_form_login = False # Find and fill username @@ -362,21 +366,21 @@ class ScreenshotService: if not used_direct_form_login: raise RuntimeError("Could not find username input field on login page") username_locator = None - + if username_locator is not None: logger.info("[DEBUG] Filling username field") await username_locator.fill(self.env.username) - + # Find and fill password password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None if username_locator is not None and not password_locator: raise RuntimeError("Could not find password input field on login page") - + if password_locator is not None: logger.info("[DEBUG] Filling password field") await password_locator.fill(self.env.password) - + # Click submit submit_locator = await self._find_submit_locator(page) if username_locator is not None else None @@ -386,14 +390,14 @@ class ScreenshotService: if submit_locator is not None: logger.info("[DEBUG] Clicking submit button") await submit_locator.click() - + # Wait for navigation after login if not used_direct_form_login: try: await page.wait_for_load_state("load", timeout=30000) except Exception as load_wait_error: logger.warning(f"[DEBUG] Login post-submit load wait timed out: {load_wait_error}") - + # Check if login was successful if not used_direct_form_login and "/login" in page.url: # Check for error messages on page @@ -402,31 +406,31 @@ class ScreenshotService: debug_path = output_path.replace(".png", "_debug_failed_login.png") await page.screenshot(path=debug_path) raise RuntimeError(f"Login failed: {error_msg}. Debug screenshot saved to {debug_path}") - + logger.info(f"[DEBUG] Login successful. Current URL: {page.url}") - + # Check cookies after successful login page_cookies = await context.cookies() logger.info(f"[DEBUG] Cookies after login: {len(page_cookies)}") for c in page_cookies: logger.info(f"[DEBUG] Cookie: name={c['name']}, domain={c['domain']}, value={c.get('value', '')[:20]}...") - + except Exception as e: page_title = await page.title() - logger.error(f"UI Login failed. Page title: {page_title}, URL: {page.url}, Error: {str(e)}") + logger.error(f"UI Login failed. Page title: {page_title}, URL: {page.url}, Error: {e!s}") debug_path = output_path.replace(".png", "_debug_failed_login.png") await page.screenshot(path=debug_path) - raise RuntimeError(f"Login failed: {str(e)}. Debug screenshot saved to {debug_path}") + raise RuntimeError(f"Login failed: {e!s}. Debug screenshot saved to {debug_path}") # 2. Navigate to dashboard # @UX_STATE: [Navigating] -> Loading dashboard UI dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true" - + if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"): dashboard_url = dashboard_url.replace("http://", "https://") logger.info(f"[DEBUG] Navigating to dashboard: {dashboard_url}") - + # Dashboard pages can keep polling/network activity open indefinitely. response = await self._goto_resilient( page, @@ -435,7 +439,7 @@ class ScreenshotService: fallback_wait_until="load", timeout=60000, ) - + if response: logger.info(f"[DEBUG] Dashboard navigation response status: {response.status}, URL: {response.url}") @@ -445,12 +449,12 @@ class ScreenshotService: raise RuntimeError( f"Dashboard navigation redirected to login page after authentication. Debug screenshot saved to {debug_path}" ) - + try: # Wait for the dashboard grid to be present await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=30000) logger.info("[DEBUG] Dashboard container loaded") - + # Wait for charts to finish loading (Superset uses loading spinners/skeletons) # We wait until loading indicators disappear or a timeout occurs try: @@ -498,7 +502,7 @@ class ScreenshotService: except Exception as e: logger.warning(f"[DEBUG] Dashboard content wait failed: {e}, proceeding anyway after delay") - + # Final stabilization delay - increased for complex dashboards logger.info("[DEBUG] Final stabilization delay...") await asyncio.sleep(15) @@ -512,42 +516,42 @@ class ScreenshotService: async def switch_tabs(depth=0): if depth > 3: return # Limit recursion depth - + tab_selectors = [ '.ant-tabs-nav-list .ant-tabs-tab', '.dashboard-component-tabs .ant-tabs-tab', '[data-test="dashboard-component-tabs"] .ant-tabs-tab' ] - + found_tabs = [] for selector in tab_selectors: found_tabs = await page.locator(selector).all() if found_tabs: break - + if found_tabs: logger.info(f"[DEBUG][TabSwitching] Found {len(found_tabs)} tabs at depth {depth}") for i, tab in enumerate(found_tabs): try: tab_text = (await tab.inner_text()).strip() tab_id = f"{depth}_{i}_{tab_text}" - + if tab_id in processed_tabs: continue - + if await tab.is_visible(): logger.info(f"[DEBUG][TabSwitching] Switching to tab: {tab_text}") processed_tabs.add(tab_id) - + is_active = "ant-tabs-tab-active" in (await tab.get_attribute("class") or "") if not is_active: await tab.click() await asyncio.sleep(2) # Wait for content to render - + await switch_tabs(depth + 1) except Exception as tab_e: logger.warning(f"[DEBUG][TabSwitching] Failed to process tab {i}: {tab_e}") - + try: first_tab = found_tabs[0] if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""): @@ -572,7 +576,7 @@ class ScreenshotService: ); }""") logger.info(f"[DEBUG] Calculated full height: {full_height}") - + # DIAGNOSTIC: Count chart elements before resize chart_count_before = await page.evaluate("""() => { return { @@ -583,7 +587,7 @@ class ScreenshotService: }; }""") logger.info(f"[DIAGNOSTIC] Chart elements BEFORE viewport resize: {chart_count_before}") - + # DIAGNOSTIC: Capture pre-resize screenshot for comparison pre_resize_path = output_path.replace(".png", "_preresize.png") try: @@ -593,15 +597,15 @@ class ScreenshotService: logger.info(f"[DIAGNOSTIC] Pre-resize screenshot saved: {pre_resize_path} ({pre_resize_size} bytes)") except Exception as pre_e: logger.warning(f"[DIAGNOSTIC] Failed to capture pre-resize screenshot: {pre_e}") - + logger.info(f"[DIAGNOSTIC] Resizing viewport from current to 1920x{int(full_height)}") await page.set_viewport_size({"width": 1920, "height": int(full_height)}) - + # DIAGNOSTIC: Increased wait time and log timing logger.info("[DIAGNOSTIC] Waiting 10 seconds after viewport resize for re-render...") await asyncio.sleep(10) logger.info("[DIAGNOSTIC] Wait completed") - + # DIAGNOSTIC: Count chart elements after resize and wait chart_count_after = await page.evaluate("""() => { return { @@ -612,7 +616,7 @@ class ScreenshotService: }; }""") logger.info(f"[DIAGNOSTIC] Chart elements AFTER viewport resize + wait: {chart_count_after}") - + # DIAGNOSTIC: Check if any charts have error states chart_errors = await page.evaluate("""() => { const errors = []; @@ -633,31 +637,31 @@ class ScreenshotService: # @UX_STATE: [Capturing] -> Executing CDP screenshot logger.info("[DEBUG] Attempting full-page screenshot via CDP...") cdp = await page.context.new_cdp_session(page) - + screenshot_data = await cdp.send("Page.captureScreenshot", { "format": "png", "fromSurface": True, "captureBeyondViewport": True }) - + image_data = base64.b64decode(screenshot_data["data"]) - + with open(output_path, 'wb') as f: f.write(image_data) - + # DIAGNOSTIC: Verify screenshot file import os final_size = os.path.getsize(output_path) if os.path.exists(output_path) else 0 logger.info(f"[DIAGNOSTIC] Final screenshot saved: {output_path}") logger.info(f"[DIAGNOSTIC] Final screenshot size: {final_size} bytes ({final_size / 1024:.2f} KB)") - + # DIAGNOSTIC: Get image dimensions try: with Image.open(output_path) as final_img: logger.info(f"[DIAGNOSTIC] Final screenshot dimensions: {final_img.width}x{final_img.height}") except Exception as img_err: logger.warning(f"[DIAGNOSTIC] Could not read final image dimensions: {img_err}") - + logger.info(f"Full-page screenshot saved to {output_path} (via CDP)") except Exception as e: logger.error(f"[DEBUG] Full-page/Tab capture failed: {e}") @@ -666,7 +670,7 @@ class ScreenshotService: except Exception as e2: logger.error(f"[DEBUG] Fallback screenshot also failed: {e2}") await page.screenshot(path=output_path, timeout=5000) - + await browser.close() return True # endregion ScreenshotService.capture_dashboard @@ -686,7 +690,7 @@ class LLMClient: self.api_key = normalized_key self.base_url = base_url self.default_model = default_model - + # DEBUG: Log initialization parameters (without exposing full API key) logger.info("[LLMClient.__init__] Initializing LLM client:") logger.info(f"[LLMClient.__init__] Provider Type: {provider_type}") @@ -749,14 +753,14 @@ class LLMClient: return False # Retry on rate limit errors and other exceptions return isinstance(exception, (RateLimitError, Exception)) - + @retry( stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, min=5, max=60), retry=retry_if_exception(_should_retry), reraise=True ) - async def get_json_completion(self, messages: List[Dict[str, Any]]) -> Dict[str, Any]: + async def get_json_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]: with belief_scope("get_json_completion"): response = None try: @@ -788,22 +792,22 @@ class LLMClient: or "response_format" in str(e).lower() or "400" in str(e) ): - logger.warning(f"[get_json_completion] JSON mode failed or not supported: {str(e)}. Falling back to plain text response.") + logger.warning(f"[get_json_completion] JSON mode failed or not supported: {e!s}. Falling back to plain text response.") response = await self.client.chat.completions.create( model=self.default_model, messages=messages ) else: raise e - + logger.debug(f"[get_json_completion] LLM Response: {response}") except OpenAIAuthenticationError as e: - logger.error(f"[get_json_completion] Authentication error: {str(e)}") + logger.error(f"[get_json_completion] Authentication error: {e!s}") # Do not retry on auth errors - re-raise to stop retry raise except RateLimitError as e: - logger.warning(f"[get_json_completion] Rate limit hit: {str(e)}") - + logger.warning(f"[get_json_completion] Rate limit hit: {e!s}") + # Extract retry_delay from error metadata if available retry_delay = 5.0 # Default fallback try: @@ -811,7 +815,7 @@ class LLMClient: # The logs show 'metadata': {'raw': '...'} which suggests a proxy or specific client wrapper # Let's try to find the 'retryDelay' in the error message or response import re - + # Try to find "retryDelay": "XXs" in the string representation of the error error_str = str(e) match = re.search(r'"retryDelay":\s*"(\d+)s"', error_str) @@ -829,14 +833,14 @@ class LLMClient: break except Exception as parse_e: logger.debug(f"[get_json_completion] Failed to parse retry delay: {parse_e}") - + # Add a small safety margin (0.5s) as requested wait_time = retry_delay + 0.5 logger.info(f"[get_json_completion] Waiting for {wait_time}s before retry...") await asyncio.sleep(wait_time) raise except Exception as e: - logger.error(f"[get_json_completion] LLM call failed: {str(e)}") + logger.error(f"[get_json_completion] LLM call failed: {e!s}") raise if not response or not hasattr(response, 'choices') or not response.choices: @@ -844,7 +848,7 @@ class LLMClient: content = response.choices[0].message.content logger.debug(f"[get_json_completion] Raw content to parse: {content}") - + try: return json.loads(content) except json.JSONDecodeError: @@ -864,7 +868,7 @@ class LLMClient: # @PRE: Client is initialized with provider credentials and default_model. # @POST: Returns lightweight JSON payload when runtime auth/model path is valid. # @SIDE_EFFECT: Calls external LLM API. - async def test_runtime_connection(self) -> Dict[str, Any]: + async def test_runtime_connection(self) -> dict[str, Any]: with belief_scope("test_runtime_connection"): messages = [ { @@ -880,7 +884,7 @@ class LLMClient: # @PRE: Client is initialized with provider credentials. # @POST: Returns a list of model ID strings. # @SIDE_EFFECT: Calls external LLM API /v1/models endpoint. - async def fetch_models(self) -> List[str]: + async def fetch_models(self) -> list[str]: with belief_scope("LLMClient.fetch_models"): try: response = await self.client.models.list() @@ -906,9 +910,9 @@ class LLMClient: async def analyze_dashboard( self, screenshot_path: str, - logs: List[str], + logs: list[str], prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"], - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("analyze_dashboard"): # Optimize image to reduce token count (US1 / T023) # Gemini/Gemma models have limits on input tokens, and large images contribute significantly. @@ -917,7 +921,7 @@ class LLMClient: # Convert to RGB if necessary if img.mode in ("RGBA", "P"): img = img.convert("RGB") - + # Resize if too large (max 1024px width while maintaining aspect ratio) # We reduce width further to 1024px to stay within token limits for long dashboards max_width = 1024 @@ -929,7 +933,7 @@ class LLMClient: new_height = int(img.height * scale) img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) logger.info(f"[analyze_dashboard] Resized image from {img.width}x{img.height} to {new_width}x{new_height}") - + # Compress and convert to base64 buffer = io.BytesIO() # Lower quality to 60% to further reduce payload size @@ -943,7 +947,7 @@ class LLMClient: log_text = "\n".join(logs) prompt = render_prompt(prompt_template, {"logs": log_text}) - + messages = [ { "role": "user", @@ -958,14 +962,14 @@ class LLMClient: ] } ] - + try: return await self.get_json_completion(messages) except Exception as e: - logger.error(f"[analyze_dashboard] Failed to get analysis: {str(e)}") + logger.error(f"[analyze_dashboard] Failed to get analysis: {e!s}") return { "status": "UNKNOWN", - "summary": f"Failed to get response from LLM: {str(e)}", + "summary": f"Failed to get response from LLM: {e!s}", "issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}] } # endregion LLMClient.analyze_dashboard diff --git a/backend/src/plugins/mapper.py b/backend/src/plugins/mapper.py index 580f4309..7e8f20c7 100644 --- a/backend/src/plugins/mapper.py +++ b/backend/src/plugins/mapper.py @@ -4,14 +4,16 @@ # @RELATION Inherits from PluginBase. Uses DatasetMapper from superset_tool. # @RELATION USES -> TaskContext -from typing import Dict, Any, Optional +from typing import Any + +from ..core.database import SessionLocal +from ..core.logger import belief_scope, logger from ..core.plugin_base import PluginBase from ..core.superset_client import SupersetClient -from ..core.logger import logger, belief_scope -from ..core.database import SessionLocal -from ..models.connection import ConnectionConfig -from ..core.utils.dataset_mapper import DatasetMapper from ..core.task_manager.context import TaskContext +from ..core.utils.dataset_mapper import DatasetMapper +from ..models.connection import ConnectionConfig + # #region MapperPlugin [TYPE Class] # @BRIEF Plugin for mapping dataset columns verbose names. @@ -78,7 +80,7 @@ class MapperPlugin(PluginBase): # @PRE: Plugin instance exists. # @POST: Returns dictionary schema. # @RETURN: Dict[str, Any] - JSON schema. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("get_schema"): return { "type": "object", @@ -132,19 +134,19 @@ class MapperPlugin(PluginBase): # @PRE: Params contain valid 'env', 'dataset_id', and 'source'. params must be a dictionary. # @POST: Updates the dataset in Superset. # @RETURN: Dict[str, Any] - Execution status. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]: + async def execute(self, params: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]: with belief_scope("execute"): env_name = params.get("env") dataset_id = params.get("dataset_id") source = params.get("source") - + # Use TaskContext logger if available, otherwise fall back to app logger log = context.logger if context else logger - + # Create sub-loggers for different components superset_log = log.with_source("superset_api") if context else log db_log = log.with_source("postgres") if context else log - + if not env_name or dataset_id is None or not source: log.error("Missing required parameters: env, dataset_id, source") raise ValueError("Missing required parameters: env, dataset_id, source") @@ -166,7 +168,7 @@ class MapperPlugin(PluginBase): if not connection_id: log.error("connection_id is required for postgres source.") raise ValueError("connection_id is required for postgres source.") - + # Load connection from DB db = SessionLocal() try: @@ -174,7 +176,7 @@ class MapperPlugin(PluginBase): if not conn_config: db_log.error(f"Connection {connection_id} not found.") raise ValueError(f"Connection {connection_id} not found.") - + postgres_config = { 'dbname': conn_config.database, 'user': conn_config.username, @@ -187,9 +189,9 @@ class MapperPlugin(PluginBase): db.close() log.info(f"Starting mapping for dataset {dataset_id} in {env_name}") - + mapper = DatasetMapper() - + try: mapper.run_mapping( superset_client=client, @@ -208,4 +210,4 @@ class MapperPlugin(PluginBase): # endregion execute # #endregion MapperPlugin -# #endregion MapperPluginModule \ No newline at end of file +# #endregion MapperPluginModule diff --git a/backend/src/plugins/migration.py b/backend/src/plugins/migration.py index 873d63fe..8ffae425 100755 --- a/backend/src/plugins/migration.py +++ b/backend/src/plugins/migration.py @@ -12,19 +12,21 @@ # @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set] # @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution. -from typing import Dict, Any, Optional import re +from typing import Any +from ..core.database import SessionLocal +from ..core.logger import belief_scope +from ..core.logger import logger as app_logger +from ..core.mapping_service import IdMappingService +from ..core.migration_engine import MigrationEngine from ..core.plugin_base import PluginBase -from ..core.logger import belief_scope, logger as app_logger from ..core.superset_client import SupersetClient +from ..core.task_manager.context import TaskContext from ..core.utils.fileio import create_temp_file from ..dependencies import get_config_manager -from ..core.migration_engine import MigrationEngine -from ..core.database import SessionLocal from ..models.mapping import DatabaseMapping, Environment -from ..core.mapping_service import IdMappingService -from ..core.task_manager.context import TaskContext + # #region MigrationPlugin [TYPE Class] # @BRIEF Implementation of the migration plugin workflow and transformation orchestration. @@ -101,12 +103,12 @@ class MigrationPlugin(PluginBase): # @PRE: ConfigManager is accessible and environments are defined. # @POST: Returns a JSON Schema dict matching current system environments. # @RETURN: Dict[str, Any] - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("MigrationPlugin.get_schema"): app_logger.reason("Generating migration UI schema") config_manager = get_config_manager() envs = [e.name for e in config_manager.get_environments()] - + schema = { "type": "object", "properties": { @@ -164,20 +166,20 @@ class MigrationPlugin(PluginBase): # @TEST_EDGE: missing_env_field -> [ValueError: Could not resolve source or target environment] # @TEST_EDGE: invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully] # @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS] - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): + async def execute(self, params: dict[str, Any], context: TaskContext | None = None): with belief_scope("MigrationPlugin.execute"): app_logger.reason("Evaluating migration task parameters", extra={"params": params}) - + source_env_id = params.get("source_env_id") target_env_id = params.get("target_env_id") selected_ids = params.get("selected_ids") - + from_env_name = params.get("from_env") to_env_name = params.get("to_env") dashboard_regex = params.get("dashboard_regex") replace_db_config = params.get("replace_db_config", False) fix_cross_filters = params.get("fix_cross_filters", True) - + task_id = params.get("_task_id") from ..dependencies import get_task_manager tm = get_task_manager() @@ -185,17 +187,17 @@ class MigrationPlugin(PluginBase): log = context.logger if context else app_logger superset_log = log.with_source("superset_api") if context else log migration_log = log.with_source("migration") if context else log - + log.info("Starting migration task.") try: config_manager = get_config_manager() environments = config_manager.get_environments() - + # Resolve environments src_env = next((e for e in environments if e.id == source_env_id), None) if source_env_id else next((e for e in environments if e.name == from_env_name), None) tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None) - + if not src_env or not tgt_env: app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name}) raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}") @@ -204,7 +206,7 @@ class MigrationPlugin(PluginBase): to_env_name = tgt_env.name app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name}) - + migration_result = { "status": "SUCCESS", "source_environment": from_env_name, @@ -217,12 +219,12 @@ class MigrationPlugin(PluginBase): from_c = SupersetClient(src_env) to_c = SupersetClient(tgt_env) - + if not from_c or not to_c: raise ValueError(f"Clients not initialized for environments: {from_env_name}, {to_env_name}") _, all_dashboards = from_c.get_dashboards() - + # Selection Logic if selected_ids: dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids] @@ -245,14 +247,14 @@ class MigrationPlugin(PluginBase): db_mapping = params.get("db_mappings", {}) if not isinstance(db_mapping, dict): db_mapping = {} - + if replace_db_config: app_logger.reason("Fetching environment DB mappings from catalog") db = SessionLocal() try: src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first() tgt_env_db = db.query(Environment).filter(Environment.name == to_env_name).first() - + if src_env_db and tgt_env_db: stored_mappings = db.query(DatabaseMapping).filter( DatabaseMapping.source_env_id == src_env_db.id, @@ -272,60 +274,59 @@ class MigrationPlugin(PluginBase): for dash in dashboards_to_migrate: dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"] app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id}) - + try: exported_content, _ = from_c.export_dashboard(dash_id) - with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path: - with create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip: - - success = engine.transform_zip( - str(tmp_zip_path), - str(tmp_new_zip), - db_mapping, - strip_databases=False, - target_env_id=tgt_env.id if tgt_env else None, - fix_cross_filters=fix_cross_filters - ) - - if not success and replace_db_config: - if task_id: - app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id}) - await tm.wait_for_resolution(task_id) - - app_logger.reason("Task resumed, re-evaluating mapping states") - db = SessionLocal() - try: - src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first() - tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first() - mappings = db.query(DatabaseMapping).filter( - DatabaseMapping.source_env_id == src_env_rt.id, - DatabaseMapping.target_env_id == tgt_env_rt.id - ).all() - db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings} - finally: - db.close() - - success = engine.transform_zip( - str(tmp_zip_path), - str(tmp_new_zip), - db_mapping, - strip_databases=False, - target_env_id=tgt_env.id if tgt_env else None, - fix_cross_filters=fix_cross_filters - ) + with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path, create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip: + + success = engine.transform_zip( + str(tmp_zip_path), + str(tmp_new_zip), + db_mapping, + strip_databases=False, + target_env_id=tgt_env.id if tgt_env else None, + fix_cross_filters=fix_cross_filters + ) + + if not success and replace_db_config: + if task_id: + app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id}) + await tm.wait_for_resolution(task_id) + + app_logger.reason("Task resumed, re-evaluating mapping states") + db = SessionLocal() + try: + src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first() + tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first() + mappings = db.query(DatabaseMapping).filter( + DatabaseMapping.source_env_id == src_env_rt.id, + DatabaseMapping.target_env_id == tgt_env_rt.id + ).all() + db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings} + finally: + db.close() + + success = engine.transform_zip( + str(tmp_zip_path), + str(tmp_new_zip), + db_mapping, + strip_databases=False, + target_env_id=tgt_env.id if tgt_env else None, + fix_cross_filters=fix_cross_filters + ) + + if success: + app_logger.reason("Pushing transformed ZIP to target Superset") + to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug) + migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) + app_logger.reflect("Import successful", extra={"title": title}) + else: + app_logger.explore("Transformation strictly failed, bypassing ingestion") + migration_log.error(f"Failed to transform ZIP for dashboard {title}") + migration_result["failed_dashboards"].append({ + "id": dash_id, "title": title, "error": "Failed to transform ZIP" + }) - if success: - app_logger.reason("Pushing transformed ZIP to target Superset") - to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug) - migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) - app_logger.reflect("Import successful", extra={"title": title}) - else: - app_logger.explore("Transformation strictly failed, bypassing ingestion") - migration_log.error(f"Failed to transform ZIP for dashboard {title}") - migration_result["failed_dashboards"].append({ - "id": dash_id, "title": title, "error": "Failed to transform ZIP" - }) - except Exception as exc: error_msg = str(exc) if "Must provide a password for the database" in error_msg: @@ -338,19 +339,19 @@ class MigrationPlugin(PluginBase): if match_alt: db_name = match_alt.group(1) - app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name}) - + app_logger.explore("Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name}) + if task_id: tm.await_input(task_id, { "type": "database_password", "databases": [db_name], "error_message": error_msg }) - + await tm.wait_for_input(task_id) task = tm.get_task(task_id) passwords = task.params.get("passwords", {}) - + if passwords: app_logger.reason(f"Retrying import for {title} with injected credentials") to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords) @@ -385,4 +386,4 @@ class MigrationPlugin(PluginBase): raise e # endregion execute # #endregion MigrationPlugin -# #endregion MigrationPlugin \ No newline at end of file +# #endregion MigrationPlugin diff --git a/backend/src/plugins/search.py b/backend/src/plugins/search.py index be976811..f5def0ad 100644 --- a/backend/src/plugins/search.py +++ b/backend/src/plugins/search.py @@ -5,12 +5,14 @@ # @RELATION USES -> TaskContext import re -from typing import Dict, Any, Optional +from typing import Any + +from ..core.logger import belief_scope, logger from ..core.plugin_base import PluginBase from ..core.superset_client import SupersetClient -from ..core.logger import logger, belief_scope from ..core.task_manager.context import TaskContext + # #region SearchPlugin [TYPE Class] # @BRIEF Plugin for searching text patterns in Superset datasets. class SearchPlugin(PluginBase): @@ -76,7 +78,7 @@ class SearchPlugin(PluginBase): # @PRE: Plugin instance exists. # @POST: Returns dictionary schema. # @RETURN: Dict[str, Any] - JSON schema. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("get_schema"): return { "type": "object", @@ -103,18 +105,18 @@ class SearchPlugin(PluginBase): # @PRE: Params contain valid 'env' and 'query'. # @POST: Returns a dictionary with count and results list. # @RETURN: Dict[str, Any] - Search results. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]: + async def execute(self, params: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]: with belief_scope("SearchPlugin.execute", f"params={params}"): env_name = params.get("env") search_query = params.get("query") - + # Use TaskContext logger if available, otherwise fall back to app logger log = context.logger if context else logger - + # Create sub-loggers for different components log.with_source("superset_api") if context else log search_log = log.with_source("search") if context else log - + if not env_name or not search_query: log.error("Missing required parameters: env, query") raise ValueError("Missing required parameters: env, query") @@ -131,7 +133,7 @@ class SearchPlugin(PluginBase): client.authenticate() log.info(f"Searching for pattern: '{search_query}' in environment: {env_name}") - + try: # Ported logic from search_script.py _, datasets = client.get_datasets(query={"columns": ["id", "table_name", "sql", "database", "columns"]}) @@ -142,7 +144,7 @@ class SearchPlugin(PluginBase): pattern = re.compile(search_query, re.IGNORECASE) results = [] - + for dataset in datasets: dataset_id = dataset.get('id') dataset_name = dataset.get('table_name', 'Unknown') @@ -190,14 +192,14 @@ class SearchPlugin(PluginBase): with belief_scope("_get_context"): if not match_text: return text[:100] + "..." if len(text) > 100 else text - + lines = text.splitlines() match_line_index = -1 for i, line in enumerate(lines): if match_text in line: match_line_index = i break - + if match_line_index != -1: start = max(0, match_line_index - context_lines) end = min(len(lines), match_line_index + context_lines + 1) @@ -209,9 +211,9 @@ class SearchPlugin(PluginBase): else: context.append(f" {line_content}") return "\n".join(context) - + return text[:100] + "..." if len(text) > 100 else text # endregion _get_context # #endregion SearchPlugin -# #endregion SearchPluginModule \ No newline at end of file +# #endregion SearchPluginModule diff --git a/backend/src/plugins/storage/plugin.py b/backend/src/plugins/storage/plugin.py index dc777f14..784c23d0 100644 --- a/backend/src/plugins/storage/plugin.py +++ b/backend/src/plugins/storage/plugin.py @@ -10,16 +10,18 @@ import os import shutil -from pathlib import Path from datetime import datetime -from typing import Dict, Any, List, Optional +from pathlib import Path +from typing import Any + from fastapi import UploadFile -from ...core.plugin_base import PluginBase from ...core.logger import belief_scope, logger -from ...models.storage import StoredFile, FileCategory -from ...dependencies import get_config_manager +from ...core.plugin_base import PluginBase from ...core.task_manager.context import TaskContext +from ...dependencies import get_config_manager +from ...models.storage import FileCategory, StoredFile + # #region StoragePlugin [TYPE Class] # @BRIEF Implementation of the storage management plugin. @@ -95,7 +97,7 @@ class StoragePlugin(PluginBase): # @PRE: None. # @POST: Returns a dictionary representing the JSON schema. # @RETURN: Dict[str, Any] - JSON schema. - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: with belief_scope("StoragePlugin:get_schema"): return { "type": "object", @@ -116,15 +118,15 @@ class StoragePlugin(PluginBase): # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: params must match the plugin schema. # @POST: Task is executed and logged. - async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): + async def execute(self, params: dict[str, Any], context: TaskContext | None = None): with belief_scope("StoragePlugin:execute"): # Use TaskContext logger if available, otherwise fall back to app logger log = context.logger if context else logger - + # Create sub-loggers for different components storage_log = log.with_source("storage") if context else log log.with_source("filesystem") if context else log - + storage_log.info(f"Executing with params: {params}") # endregion execute @@ -136,10 +138,10 @@ class StoragePlugin(PluginBase): with belief_scope("StoragePlugin:get_storage_root"): config_manager = get_config_manager() global_settings = config_manager.get_config().settings - + # Use storage.root_path as the source of truth for storage UI root = Path(global_settings.storage.root_path) - + if not root.is_absolute(): # Resolve relative to the backend directory # Path(__file__) is backend/src/plugins/storage/plugin.py @@ -157,7 +159,7 @@ class StoragePlugin(PluginBase): # @PRE: pattern must be a valid format string. # @POST: Returns the resolved path string. # @RETURN: str - The resolved path. - def resolve_path(self, pattern: str, variables: Dict[str, str]) -> str: + def resolve_path(self, pattern: str, variables: dict[str, str]) -> str: with belief_scope("StoragePlugin:resolve_path"): # Add common variables vars_with_defaults = { @@ -215,10 +217,10 @@ class StoragePlugin(PluginBase): # @RETURN: List[StoredFile] - List of file and directory metadata objects. def list_files( self, - category: Optional[FileCategory] = None, - subpath: Optional[str] = None, + category: FileCategory | None = None, + subpath: str | None = None, recursive: bool = False, - ) -> List[StoredFile]: + ) -> list[StoredFile]: with belief_scope("StoragePlugin:list_files"): root = self.get_storage_root() logger.reason( @@ -245,9 +247,9 @@ class StoragePlugin(PluginBase): ) ) return sorted(files, key=lambda x: x.name) - + categories = [category] if category else list(FileCategory) - + for cat in categories: # Scan the category subfolder + optional subpath base_dir = root / cat.value @@ -258,7 +260,7 @@ class StoragePlugin(PluginBase): if not target_dir.exists(): continue - + logger.reason(f"Scanning directory: {target_dir}", extra={"src": "StoragePlugin.list_files"}) if recursive: @@ -299,7 +301,7 @@ class StoragePlugin(PluginBase): category=cat, mime_type="directory" if is_dir else None )) - + # Sort: directories first, then by name return sorted(files, key=lambda x: (x.mime_type != "directory", x.name)) # endregion list_files @@ -313,20 +315,20 @@ class StoragePlugin(PluginBase): # @POST: File is written to disk and metadata is returned. # @RETURN: StoredFile - Metadata of the saved file. # @SIDE_EFFECT: Writes file to disk. - async def save_file(self, file: UploadFile, category: FileCategory, subpath: Optional[str] = None) -> StoredFile: + async def save_file(self, file: UploadFile, category: FileCategory, subpath: str | None = None) -> StoredFile: with belief_scope("StoragePlugin:save_file"): root = self.get_storage_root() dest_dir = root / category.value if subpath: dest_dir = dest_dir / subpath - + dest_dir.mkdir(parents=True, exist_ok=True) - + dest_path = self.validate_path(dest_dir / file.filename) - + with dest_path.open("wb") as buffer: shutil.copyfileobj(file.file, buffer) - + stat = dest_path.stat() return StoredFile( name=dest_path.name, @@ -350,7 +352,7 @@ class StoragePlugin(PluginBase): root = self.get_storage_root() # path is relative to root, but we ensure it starts with category full_path = self.validate_path(root / path) - + if not str(Path(path)).startswith(category.value): raise ValueError(f"Path {path} does not belong to category {category}") @@ -375,13 +377,13 @@ class StoragePlugin(PluginBase): with belief_scope("StoragePlugin:get_file_path"): root = self.get_storage_root() file_path = self.validate_path(root / path) - + if not str(Path(path)).startswith(category.value): raise ValueError(f"Path {path} does not belong to category {category}") if not file_path.exists() or file_path.is_dir(): raise FileNotFoundError(f"File {path} not found") - + return file_path # endregion get_file_path diff --git a/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py b/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py index 9b980d2a..6ce0d255 100644 --- a/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py +++ b/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py @@ -15,8 +15,8 @@ from unittest.mock import MagicMock, patch from src.plugins.translate.sql_generator import ( SQLGenerator, - _normalize_timestamp_value, _encode_sql_value, + _normalize_timestamp_value, ) @@ -363,22 +363,21 @@ class TestOrchestratorInsertFlow: orch = TranslationOrchestrator(db, config_manager, "test-user") # Mock the event_log and SupersetSqlLabExecutor - with patch.object(orch, "event_log") as mock_event_log: - with patch( - "src.plugins.translate.orchestrator.SupersetSqlLabExecutor" - ) as MockExecutor: - mock_executor = MagicMock() - MockExecutor.return_value = mock_executor - mock_executor.resolve_database_id.return_value = None - mock_executor.get_database_backend.return_value = None - mock_executor.execute_and_poll.return_value = { - "status": "success", - "query_id": "q-123", - "rows_affected": 2, - } + with patch.object(orch, "event_log") as mock_event_log, patch( + "src.plugins.translate.orchestrator.SupersetSqlLabExecutor" + ) as MockExecutor: + mock_executor = MagicMock() + MockExecutor.return_value = mock_executor + mock_executor.resolve_database_id.return_value = None + mock_executor.get_database_backend.return_value = None + mock_executor.execute_and_poll.return_value = { + "status": "success", + "query_id": "q-123", + "rows_affected": 2, + } - # Call _generate_and_insert_sql - result = orch._generate_and_insert_sql(job, run) + # Call _generate_and_insert_sql + result = orch._generate_and_insert_sql(job, run) # Verify executor was called mock_executor.execute_and_poll.assert_called_once() diff --git a/backend/src/plugins/translate/__tests__/test_dictionary.py b/backend/src/plugins/translate/__tests__/test_dictionary.py index 9ab0277f..524d035a 100644 --- a/backend/src/plugins/translate/__tests__/test_dictionary.py +++ b/backend/src/plugins/translate/__tests__/test_dictionary.py @@ -17,23 +17,18 @@ # @TEST_EDGE: import_invalid_format -> ValueError for missing columns # @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry] + import pytest -import csv -import io -from datetime import datetime, timezone from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker, Session +from sqlalchemy.orm import Session, sessionmaker from src.models.translate import ( Base, - TerminologyDictionary, - DictionaryEntry, TranslationJob, TranslationJobDictionary, - TranslationRun, ) +from src.plugins.translate._utils import _detect_delimiter, _normalize_term from src.plugins.translate.dictionary import DictionaryManager -from src.plugins.translate._utils import _normalize_term, _detect_delimiter # region _FakeJob [TYPE Class] diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py index 6d57ba25..31f38a96 100644 --- a/backend/src/plugins/translate/__tests__/test_orchestrator.py +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -11,18 +11,19 @@ # @TEST_EDGE: invalid_run_status -> raises ValueError # @TEST_EDGE: executor_failure -> run is marked FAILED -import pytest +from datetime import UTC, datetime from unittest.mock import MagicMock, patch -from datetime import datetime, timezone -from src.plugins.translate.orchestrator import TranslationOrchestrator -from src.plugins.translate.events import TranslationEventLog +import pytest + from src.models.translate import ( TranslationJob, - TranslationRun, - TranslationRecord, TranslationPreviewSession, + TranslationRecord, + TranslationRun, ) +from src.plugins.translate.events import TranslationEventLog +from src.plugins.translate.orchestrator import TranslationOrchestrator # region mock_job [TYPE Function] @@ -63,7 +64,7 @@ def mock_preview_session() -> MagicMock: session.id = "session-1" session.job_id = "job-123" session.status = "APPLIED" - session.created_at = datetime.now(timezone.utc) + session.created_at = datetime.now(UTC) return session @@ -251,7 +252,7 @@ class TestTranslationOrchestrator: completed_run_response.successful_records = 10 completed_run_response.failed_records = 0 completed_run_response.skipped_records = 0 - completed_run_response.completed_at = datetime.now(timezone.utc) + completed_run_response.completed_at = datetime.now(UTC) completed_run_response.error_message = None completed_run_response.insert_status = "success" completed_run_response.superset_execution_id = "q-1" @@ -276,12 +277,11 @@ class TestTranslationOrchestrator: MockExecutor.return_value = mock_executor_instance orch = TranslationOrchestrator(db, config_manager, "test-user") - with patch.object(orch, "event_log"): - with patch.object( - orch, "_generate_and_insert_sql", - return_value={"status": "success", "query_id": "q-1", "rows_affected": 10}, - ): - result = orch.execute_run(run) + with patch.object(orch, "event_log"), patch.object( + orch, "_generate_and_insert_sql", + return_value={"status": "success", "query_id": "q-1", "rows_affected": 10}, + ): + result = orch.execute_run(run) assert result.status == "COMPLETED" assert result.insert_status == "success" @@ -346,7 +346,7 @@ class TestTranslationOrchestrator: completed_run.successful_records = 5 completed_run.failed_records = 0 completed_run.skipped_records = 0 - completed_run.completed_at = datetime.now(timezone.utc) + completed_run.completed_at = datetime.now(UTC) completed_run.error_message = None with patch( @@ -357,9 +357,8 @@ class TestTranslationOrchestrator: MockExecutor.return_value = mock_executor_instance orch = TranslationOrchestrator(db, config_manager, "test-user") - with patch.object(orch, "_generate_and_insert_sql") as mock_gen_sql: - with patch.object(orch, "event_log"): - result = orch.execute_run(run, skip_insert=True) + with patch.object(orch, "_generate_and_insert_sql") as mock_gen_sql, patch.object(orch, "event_log"): + result = orch.execute_run(run, skip_insert=True) assert result.status == "COMPLETED" # _generate_and_insert_sql should NOT be called in skip_insert mode @@ -403,7 +402,7 @@ class TestTranslationOrchestrator: completed_run_response.successful_records = 3 completed_run_response.failed_records = 0 completed_run_response.skipped_records = 0 - completed_run_response.completed_at = datetime.now(timezone.utc) + completed_run_response.completed_at = datetime.now(UTC) completed_run_response.error_message = None completed_run_response.insert_status = "failed" completed_run_response.superset_execution_id = "" @@ -429,9 +428,8 @@ class TestTranslationOrchestrator: with patch.object( orch, "_generate_and_insert_sql", return_value={"status": "failed", "error_message": "timeout", "query_id": None}, - ): - with patch.object(orch, "event_log"): - result = orch.execute_run(run) + ), patch.object(orch, "event_log"): + result = orch.execute_run(run) assert result.status == "COMPLETED" assert result.insert_status == "failed" @@ -504,8 +502,8 @@ class TestTranslationOrchestrator: run.id = "run-1" run.job_id = "job-123" run.status = "COMPLETED" - run.started_at = datetime.now(timezone.utc) - run.completed_at = datetime.now(timezone.utc) + run.started_at = datetime.now(UTC) + run.completed_at = datetime.now(UTC) run.error_message = None run.total_records = 100 run.successful_records = 95 @@ -514,7 +512,7 @@ class TestTranslationOrchestrator: run.insert_status = "success" run.superset_execution_id = "query-1" run.created_by = "test-user" - run.created_at = datetime.now(timezone.utc) + run.created_at = datetime.now(UTC) db.query.return_value.filter.return_value.first.return_value = run db.query.return_value.filter.return_value.count.return_value = 5 @@ -545,7 +543,7 @@ class TestTranslationOrchestrator: record.source_object_name = "" record.status = "SUCCESS" record.error_message = None - record.created_at = datetime.now(timezone.utc) + record.created_at = datetime.now(UTC) db.query.return_value.filter.return_value.count.return_value = 1 db.query.return_value.filter.return_value.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [record] @@ -567,8 +565,8 @@ class TestTranslationOrchestrator: run.id = "run-1" run.job_id = "job-123" run.status = "COMPLETED" - run.started_at = datetime.now(timezone.utc) - run.completed_at = datetime.now(timezone.utc) + run.started_at = datetime.now(UTC) + run.completed_at = datetime.now(UTC) run.error_message = None run.total_records = 100 run.successful_records = 95 @@ -577,7 +575,7 @@ class TestTranslationOrchestrator: run.insert_status = "success" run.superset_execution_id = "query-1" run.created_by = "test-user" - run.created_at = datetime.now(timezone.utc) + run.created_at = datetime.now(UTC) db.query.return_value.filter.return_value.count.return_value = 1 db.query.return_value.filter.return_value.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [run] diff --git a/backend/src/plugins/translate/__tests__/test_preview.py b/backend/src/plugins/translate/__tests__/test_preview.py index e48b1542..4c6c9ebf 100644 --- a/backend/src/plugins/translate/__tests__/test_preview.py +++ b/backend/src/plugins/translate/__tests__/test_preview.py @@ -4,18 +4,18 @@ # @LAYER: Test # @RELATION: BINDS_TO -> [TranslationPreview:Module] -from unittest.mock import MagicMock, patch, PropertyMock -from datetime import datetime, timezone, timedelta import json +from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock, patch + import pytest from src.models.translate import ( TranslationJob, - TranslationPreviewSession, TranslationPreviewRecord, - TranslationJobDictionary, + TranslationPreviewSession, ) -from src.plugins.translate.preview import TranslationPreview, TokenEstimator +from src.plugins.translate.preview import TokenEstimator, TranslationPreview # region _make_mock_job [TYPE Function] @@ -234,7 +234,7 @@ class TestTranslationPreview: session.id = "session-1" session.job_id = "job-123" session.status = "ACTIVE" - session.created_at = datetime.now(timezone.utc) + session.created_at = datetime.now(UTC) # Mock record record = MagicMock(spec=TranslationPreviewRecord) @@ -371,8 +371,8 @@ class TestTranslationPreview: session.job_id = "job-123" session.status = "ACTIVE" session.created_by = "test-user" - session.created_at = datetime.now(timezone.utc) - session.expires_at = datetime.now(timezone.utc) + timedelta(hours=24) + session.created_at = datetime.now(UTC) + session.expires_at = datetime.now(UTC) + timedelta(hours=24) record = MagicMock(spec=TranslationPreviewRecord) record.id = "record-1" diff --git a/backend/src/plugins/translate/__tests__/test_sql_generator.py b/backend/src/plugins/translate/__tests__/test_sql_generator.py index 6afc040f..936d31dc 100644 --- a/backend/src/plugins/translate/__tests__/test_sql_generator.py +++ b/backend/src/plugins/translate/__tests__/test_sql_generator.py @@ -9,13 +9,14 @@ # @TEST_EDGE: null_values -> properly encoded as NULL # @TEST_EDGE: sql_injection -> values with single quotes are escaped +from typing import Any + import pytest -from typing import Any, Dict, List from src.plugins.translate.sql_generator import ( SQLGenerator, - _quote_identifier, _encode_sql_value, + _quote_identifier, generate_insert_sql, generate_upsert_sql, ) @@ -33,7 +34,7 @@ def _normalize_sql(sql: str) -> str: # region sample_rows [TYPE Function] # @PURPOSE: Sample rows fixture for SQL generation tests. @pytest.fixture -def sample_rows() -> List[Dict[str, Any]]: +def sample_rows() -> list[dict[str, Any]]: return [ {"col1": "hello", "col2": 42, "col3": None}, {"col1": "world", "col2": 99, "col3": "note"}, @@ -129,7 +130,7 @@ class TestEncodeSqlValue: class TestGenerateInsertSql: # region test_basic_insert [TYPE Function] # @PURPOSE: Generates basic INSERT with VALUES. - def test_basic_insert(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_basic_insert(self, sample_rows: list[dict[str, Any]]) -> None: sql = generate_insert_sql( target_schema=None, target_table="my_table", @@ -145,7 +146,7 @@ class TestGenerateInsertSql: # region test_insert_with_schema [TYPE Function] # @PURPOSE: Schema-qualified table name. - def test_insert_with_schema(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_insert_with_schema(self, sample_rows: list[dict[str, Any]]) -> None: sql = generate_insert_sql( target_schema="public", target_table="my_table", @@ -157,7 +158,7 @@ class TestGenerateInsertSql: # region test_empty_columns_raises [TYPE Function] # @PURPOSE: Empty columns list raises ValueError. - def test_empty_columns_raises(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_empty_columns_raises(self, sample_rows: list[dict[str, Any]]) -> None: with pytest.raises(ValueError, match="At least one column"): generate_insert_sql( target_schema=None, @@ -218,7 +219,7 @@ class TestGenerateInsertSql: class TestGenerateUpsertSql: # region test_basic_upsert [TYPE Function] # @PURPOSE: Generates INSERT ... ON CONFLICT DO UPDATE. - def test_basic_upsert(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_basic_upsert(self, sample_rows: list[dict[str, Any]]) -> None: sql = generate_upsert_sql( target_schema=None, target_table="my_table", @@ -247,7 +248,7 @@ class TestGenerateUpsertSql: # region test_upsert_empty_keys [TYPE Function] # @PURPOSE: Empty key_columns raises ValueError. - def test_upsert_empty_keys(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_upsert_empty_keys(self, sample_rows: list[dict[str, Any]]) -> None: with pytest.raises(ValueError, match="key_columns"): generate_upsert_sql( target_schema=None, @@ -268,7 +269,7 @@ class TestGenerateUpsertSql: class TestSQLGenerator: # region test_postgresql_insert [TYPE Function] # @PURPOSE: PostgreSQL dialect generates proper INSERT. - def test_postgresql_insert(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_postgresql_insert(self, sample_rows: list[dict[str, Any]]) -> None: sql, count = SQLGenerator.generate( dialect="postgresql", target_schema="public", @@ -283,7 +284,7 @@ class TestSQLGenerator: # region test_postgresql_upsert [TYPE Function] # @PURPOSE: PostgreSQL dialect generates proper UPSERT. - def test_postgresql_upsert(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_postgresql_upsert(self, sample_rows: list[dict[str, Any]]) -> None: sql, count = SQLGenerator.generate( dialect="postgresql", target_schema="public", @@ -298,7 +299,7 @@ class TestSQLGenerator: # region test_clickhouse_insert [TYPE Function] # @PURPOSE: ClickHouse dialect generates plain INSERT. - def test_clickhouse_insert(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_clickhouse_insert(self, sample_rows: list[dict[str, Any]]) -> None: sql, count = SQLGenerator.generate( dialect="clickhouse", target_schema="default", @@ -314,7 +315,7 @@ class TestSQLGenerator: # region test_clickhouse_upsert_fallback [TYPE Function] # @PURPOSE: ClickHouse with MERGE strategy falls back to plain INSERT with warning. - def test_clickhouse_upsert_fallback(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_clickhouse_upsert_fallback(self, sample_rows: list[dict[str, Any]]) -> None: sql, count = SQLGenerator.generate( dialect="clickhouse", target_schema="default", @@ -342,7 +343,7 @@ class TestSQLGenerator: # region test_generate_batch [TYPE Function] # @PURPOSE: generate_batch splits large row sets. - def test_generate_batch(self, sample_rows: List[Dict[str, Any]]) -> None: + def test_generate_batch(self, sample_rows: list[dict[str, Any]]) -> None: # Create many rows to test batching many_rows = sample_rows * 5 # 10 rows statements = SQLGenerator.generate_batch( diff --git a/backend/src/plugins/translate/_utils.py b/backend/src/plugins/translate/_utils.py index 18578e20..21c9b1f3 100644 --- a/backend/src/plugins/translate/_utils.py +++ b/backend/src/plugins/translate/_utils.py @@ -2,7 +2,6 @@ # @BRIEF Utility helpers for dictionary management (term normalization and delimiter detection). # @LAYER: Domain -import re import unicodedata diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index 8abe22c8..d3fd9653 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -12,17 +12,19 @@ import csv import io import re -from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.orm import Session +from typing import Any + from sqlalchemy import func -from ...core.logger import logger, belief_scope +from sqlalchemy.orm import Session + +from ...core.logger import belief_scope, logger from ...models.translate import ( - TerminologyDictionary, DictionaryEntry, - TranslationJobDictionary, + TerminologyDictionary, TranslationJob, + TranslationJobDictionary, ) -from ._utils import _normalize_term, _detect_delimiter +from ._utils import _detect_delimiter, _normalize_term # #region DictionaryManager [C:4] [TYPE Class] @@ -38,7 +40,7 @@ class DictionaryManager: @staticmethod def create_dictionary( db: Session, name: str, source_dialect: str, target_dialect: str, - created_by: Optional[str] = None, description: Optional[str] = None, + created_by: str | None = None, description: str | None = None, is_active: bool = True, ) -> TerminologyDictionary: with belief_scope("DictionaryManager.create_dictionary"): @@ -64,9 +66,9 @@ class DictionaryManager: # @POST: Dictionary metadata is updated and returned. @staticmethod def update_dictionary( - db: Session, dict_id: str, name: Optional[str] = None, - description: Optional[str] = None, source_dialect: Optional[str] = None, - target_dialect: Optional[str] = None, is_active: Optional[bool] = None, + db: Session, dict_id: str, name: str | None = None, + description: str | None = None, source_dialect: str | None = None, + target_dialect: str | None = None, is_active: bool | None = None, ) -> TerminologyDictionary: with belief_scope("DictionaryManager.update_dictionary"): dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() @@ -151,7 +153,7 @@ class DictionaryManager: @staticmethod def list_dictionaries( db: Session, page: int = 1, page_size: int = 20, - ) -> Tuple[List[TerminologyDictionary], int]: + ) -> tuple[list[TerminologyDictionary], int]: total = db.query(func.count(TerminologyDictionary.id)).scalar() or 0 dictionaries = ( db.query(TerminologyDictionary) @@ -170,7 +172,7 @@ class DictionaryManager: @staticmethod def add_entry( db: Session, dict_id: str, source_term: str, target_term: str, - context_notes: Optional[str] = None, + context_notes: str | None = None, ) -> DictionaryEntry: with belief_scope("DictionaryManager.add_entry"): normalized = _normalize_term(source_term) @@ -209,8 +211,8 @@ class DictionaryManager: # @POST: Entry fields are updated. @staticmethod def edit_entry( - db: Session, entry_id: str, source_term: Optional[str] = None, - target_term: Optional[str] = None, context_notes: Optional[str] = None, + db: Session, entry_id: str, source_term: str | None = None, + target_term: str | None = None, context_notes: str | None = None, ) -> DictionaryEntry: with belief_scope("DictionaryManager.edit_entry"): entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() @@ -287,7 +289,7 @@ class DictionaryManager: @staticmethod def list_entries( db: Session, dict_id: str, page: int = 1, page_size: int = 50, - ) -> Tuple[List[DictionaryEntry], int]: + ) -> tuple[list[DictionaryEntry], int]: total = ( db.query(func.count(DictionaryEntry.id)) .filter(DictionaryEntry.dictionary_id == dict_id) @@ -313,10 +315,10 @@ class DictionaryManager: @staticmethod def import_entries( db: Session, dict_id: str, content: str, - delimiter: Optional[str] = None, + delimiter: str | None = None, on_conflict: str = "overwrite", preview_only: bool = False, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("DictionaryManager.import_entries"): # Validate dictionary dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() @@ -329,7 +331,7 @@ class DictionaryManager: logger.reason("Detected delimiter", {"delimiter": repr(delimiter)}) if delimiter not in (",", "\t"): - raise ValueError(f"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\t'.") + raise ValueError(f"Unsupported delimiter: {delimiter!r}. Use ',' or '\\t'.") # Parse content reader = csv.DictReader(io.StringIO(content), delimiter=delimiter) @@ -340,7 +342,7 @@ class DictionaryManager: f"Got: {reader.fieldnames}" ) - result: Dict[str, Any] = { + result: dict[str, Any] = { "total": 0, "created": 0, "updated": 0, @@ -451,8 +453,8 @@ class DictionaryManager: # @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. @staticmethod def filter_for_batch( - db: Session, source_texts: List[str], job_id: str, - ) -> List[Dict[str, Any]]: + db: Session, source_texts: list[str], job_id: str, + ) -> list[dict[str, Any]]: with belief_scope("DictionaryManager.filter_for_batch"): # Get dictionaries attached to this job job_dict_links = ( @@ -492,7 +494,7 @@ class DictionaryManager: if not all_entries: return [] - matched: List[Dict[str, Any]] = [] + matched: list[dict[str, Any]] = [] seen_source_match: set = set() for text_idx, source_text in enumerate(source_texts): @@ -554,11 +556,11 @@ class DictionaryManager: source_term: str, incorrect_target_term: str, corrected_target_term: str, - origin_run_id: Optional[str] = None, - origin_row_key: Optional[str] = None, - origin_user_id: Optional[str] = None, + origin_run_id: str | None = None, + origin_row_key: str | None = None, + origin_user_id: str | None = None, on_conflict: str = "overwrite", - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("DictionaryManager.submit_correction"): # Validate dictionary exists dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() @@ -575,7 +577,7 @@ class DictionaryManager: .first() ) - result: Dict[str, Any] = { + result: dict[str, Any] = { "source_term": source_term, "target_term": corrected_target_term, "action": "created", @@ -648,17 +650,17 @@ class DictionaryManager: def submit_bulk_corrections( db: Session, dict_id: str, - corrections: List[Dict[str, Any]], - origin_user_id: Optional[str] = None, - ) -> Dict[str, Any]: + corrections: list[dict[str, Any]], + origin_user_id: str | None = None, + ) -> dict[str, Any]: with belief_scope("DictionaryManager.submit_bulk_corrections"): # Validate dictionary first dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary '{dict_id}' not found") - results: List[Dict[str, Any]] = [] - conflicts: List[Dict[str, Any]] = [] + results: list[dict[str, Any]] = [] + conflicts: list[dict[str, Any]] = [] all_ok = True for corr in corrections: diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index 4c2c2b3f..059da233 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -11,13 +11,14 @@ # @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. # @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. # @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit. -from typing import Any, Dict, List, Optional -from sqlalchemy.orm import Session -from datetime import datetime, timezone, timedelta import uuid +from datetime import UTC, datetime, timedelta +from typing import Any -from ...core.logger import logger, belief_scope -from ...models.translate import TranslationEvent, MetricSnapshot +from sqlalchemy.orm import Session + +from ...core.logger import belief_scope, logger +from ...models.translate import MetricSnapshot, TranslationEvent # Terminal events per run — exactly one of these must exist for a completed run. TERMINAL_EVENT_TYPES = {"RUN_COMPLETED", "RUN_FAILED", "RUN_CANCELLED"} @@ -56,9 +57,9 @@ class TranslationEventLog: self, job_id: str, event_type: str, - payload: Optional[Dict[str, Any]] = None, - run_id: Optional[str] = None, - created_by: Optional[str] = None, + payload: dict[str, Any] | None = None, + run_id: str | None = None, + created_by: str | None = None, ) -> TranslationEvent: with belief_scope("TranslationEventLog.log_event"): if event_type not in VALID_EVENT_TYPES: @@ -106,7 +107,7 @@ class TranslationEventLog: event_type=event_type, event_data=payload or {}, created_by=created_by, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) self.db.add(event) self.db.flush() @@ -126,12 +127,12 @@ class TranslationEventLog: # @POST: Returns list of TranslationEvent dicts matching filters. def query_events( self, - job_id: Optional[str] = None, - run_id: Optional[str] = None, - event_type: Optional[str] = None, + job_id: str | None = None, + run_id: str | None = None, + event_type: str | None = None, limit: int = 100, offset: int = 0, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: with belief_scope("TranslationEventLog.query_events"): query = self.db.query(TranslationEvent) @@ -172,9 +173,9 @@ class TranslationEventLog: self, retention_days: int = DEFAULT_RETENTION_DAYS, batch_size: int = 1000, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("TranslationEventLog.prune_expired"): - cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + cutoff = datetime.now(UTC) - timedelta(days=retention_days) logger.reason("Pruning expired events", { "cutoff": cutoff.isoformat(), "retention_days": retention_days, @@ -197,7 +198,7 @@ class TranslationEventLog: key_hash=f"prune_{cutoff.timestamp():.0f}", covers_events_before=cutoff, total_records=total_expired, - snapshot_date=datetime.now(timezone.utc), + snapshot_date=datetime.now(UTC), ) self.db.add(snapshot) self.db.flush() @@ -235,7 +236,7 @@ class TranslationEventLog: # @PURPOSE: Get a summary of events for a run, including invariant check. # @PRE: run_id is not None. # @POST: Returns dict with event list and invariant validity. - def get_run_event_summary(self, run_id: str) -> Dict[str, Any]: + def get_run_event_summary(self, run_id: str) -> dict[str, Any]: with belief_scope("TranslationEventLog.get_run_event_summary"): events = self.query_events(run_id=run_id) diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index d1aeddd0..a1b72b47 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -16,27 +16,27 @@ import json import time import uuid -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Set, Tuple, Callable +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any from sqlalchemy.orm import Session -from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager +from ...core.logger import belief_scope, logger from ...models.translate import ( - TranslationJob, - TranslationRun, TranslationBatch, - TranslationRecord, - TranslationPreviewSession, + TranslationJob, TranslationPreviewRecord, + TranslationPreviewSession, + TranslationRecord, + TranslationRun, ) -from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt +from ...services.llm_provider import LLMProviderService from .dictionary import DictionaryManager from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE - # #region MAX_RETRIES_PER_BATCH [TYPE Constant] # @BRIEF Maximum number of retries for a single batch before marking it failed. MAX_RETRIES_PER_BATCH = 3 @@ -54,14 +54,14 @@ class TranslationExecutor: self, db: Session, config_manager: ConfigManager, - current_user: Optional[str] = None, - on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, + current_user: str | None = None, + on_batch_progress: Callable[[str, int, int, int, int], None] | None = None, ): self.db = db self.config_manager = config_manager self.current_user = current_user self.on_batch_progress = on_batch_progress - self._current_run_id: Optional[str] = None + self._current_run_id: str | None = None # region execute_run [TYPE Function] # @PURPOSE: Run full translation execution for a TranslationRun. @@ -71,7 +71,7 @@ class TranslationExecutor: def execute_run( self, run: TranslationRun, - llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None, + llm_progress_callback: Callable[[str, int, int, int], None] | None = None, ) -> TranslationRun: with belief_scope("TranslationExecutor.execute_run"): job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() @@ -86,7 +86,7 @@ class TranslationExecutor: # Mark run as RUNNING run.status = "RUNNING" - run.started_at = datetime.now(timezone.utc) + run.started_at = datetime.now(UTC) self.db.flush() # Fetch source rows from the accepted preview session @@ -94,7 +94,7 @@ class TranslationExecutor: if not source_rows: logger.explore("No source rows to translate", {"run_id": run.id}) run.status = "COMPLETED" - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() return run @@ -108,7 +108,7 @@ class TranslationExecutor: run.status = "COMPLETED" run.insert_status = "skipped" run.total_records = 0 - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.commit() return @@ -163,7 +163,7 @@ class TranslationExecutor: else: run.status = "COMPLETED" # Partial success - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() logger.reflect("Translation execution complete", { @@ -183,7 +183,7 @@ class TranslationExecutor: # @PRE: job_id exists. Job may have source_datasource_id for full fetch. # @POST: Returns list of dicts with source data (all rows from the source datasource). # @SIDE_EFFECT: Makes HTTP call to Superset chart data API when datasource is configured. - def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: + def _fetch_source_rows(self, job_id: str, run_id: str) -> list[dict[str, Any]]: with belief_scope("TranslationExecutor._fetch_source_rows"): job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() @@ -390,7 +390,7 @@ class TranslationExecutor: # @PURPOSE: Extract data rows from Superset chart data API response. # @POST: Returns list of dicts with column-value pairs. @staticmethod - def _extract_chart_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: + def _extract_chart_data_rows(response: dict[str, Any]) -> list[dict[str, Any]]: result = response.get("result") if isinstance(result, list): for item in result: @@ -420,8 +420,8 @@ class TranslationExecutor: job: TranslationJob, run_id: str, batch_index: int, - batch_rows: List[Dict[str, Any]], - ) -> Dict[str, int]: + batch_rows: list[dict[str, Any]], + ) -> dict[str, int]: with belief_scope("TranslationExecutor._process_batch"): batch_start = time.monotonic() @@ -432,7 +432,7 @@ class TranslationExecutor: batch_index=batch_index, status="RUNNING", total_records=len(batch_rows), - started_at=datetime.now(timezone.utc), + started_at=datetime.now(UTC), ) self.db.add(batch) self.db.flush() @@ -496,7 +496,7 @@ class TranslationExecutor: # Update batch status batch.successful_records = result["successful"] batch.failed_records = result["failed"] - batch.completed_at = datetime.now(timezone.utc) + batch.completed_at = datetime.now(UTC) batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS" self.db.flush() @@ -519,10 +519,10 @@ class TranslationExecutor: self, job: TranslationJob, run_id: str, - batch_rows: List[Dict[str, Any]], - dict_matches: List[Dict[str, Any]], + batch_rows: list[dict[str, Any]], + dict_matches: list[dict[str, Any]], batch_id: str, - ) -> Dict[str, int]: + ) -> dict[str, int]: with belief_scope("TranslationExecutor._call_llm_for_batch"): # Build dictionary section dictionary_section = "" @@ -810,7 +810,7 @@ class TranslationExecutor: # @PRE: response_text is valid JSON with {"rows": [...]} structure. # @POST: Returns dict mapping row_id to translation text. @staticmethod - def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: + def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]: with belief_scope("TranslationExecutor._parse_llm_response"): try: data = json.loads(response_text) @@ -830,7 +830,7 @@ class TranslationExecutor: if not isinstance(rows, list): raise ValueError("LLM response missing 'rows' array") - translations: Dict[str, str] = {} + translations: dict[str, str] = {} for item in rows: row_id = str(item.get("row_id", "")) translation = item.get("translation") diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index d47ea3b8..b6c161dd 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -10,15 +10,16 @@ # @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture. # @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary. -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.orm import Session -from sqlalchemy import func +from datetime import UTC, datetime +from typing import Any -from ...core.logger import logger, belief_scope +from sqlalchemy import func +from sqlalchemy.orm import Session + +from ...core.logger import belief_scope from ...models.translate import ( - TranslationEvent, MetricSnapshot, + TranslationEvent, TranslationRun, TranslationSchedule, ) @@ -35,7 +36,7 @@ class TranslationMetrics: # @PURPOSE: Get aggregated metrics for a specific job. # @PRE: job_id exists. # @POST: Returns dict with metrics from events + latest snapshot. - def get_job_metrics(self, job_id: str) -> Dict[str, Any]: + def get_job_metrics(self, job_id: str) -> dict[str, Any]: with belief_scope("TranslationMetrics.get_job_metrics"): # Run counts from TranslationRun run_counts = ( @@ -49,7 +50,7 @@ class TranslationMetrics: ) total_runs = 0 - status_counts: Dict[str, int] = {} + status_counts: dict[str, int] = {} for status_val, count in run_counts: status_counts[status_val] = count total_runs += count @@ -121,7 +122,7 @@ class TranslationMetrics: pass # Live events (<90 days) for token/cost - cutoff = datetime.now(timezone.utc) + cutoff = datetime.now(UTC) live_events = ( self.db.query(TranslationEvent) .filter( @@ -153,7 +154,7 @@ class TranslationMetrics: # region get_all_metrics [TYPE Function] # @PURPOSE: Get aggregated metrics for all jobs. # @POST: Returns list of per-job metrics. - def get_all_metrics(self) -> List[Dict[str, Any]]: + def get_all_metrics(self) -> list[dict[str, Any]]: with belief_scope("TranslationMetrics.get_all_metrics"): job_ids = ( self.db.query(TranslationRun.job_id) diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index f9196eeb..59531e3d 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -18,28 +18,26 @@ # @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. import json -import time import uuid -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Callable, Tuple +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any from sqlalchemy.orm import Session -from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager +from ...core.logger import belief_scope, logger from ...models.translate import ( - TranslationJob, - TranslationRun, TranslationBatch, - TranslationRecord, + TranslationJob, TranslationPreviewSession, + TranslationRecord, + TranslationRun, ) -from ...schemas.translate import TranslationRunResponse +from .events import TranslationEventLog from .executor import TranslationExecutor from .sql_generator import SQLGenerator from .superset_executor import SupersetSqlLabExecutor -from .events import TranslationEventLog -from ..translate.service import TranslateJobService # #region TranslationOrchestrator [C:5] [TYPE Class] @@ -54,13 +52,13 @@ class TranslationOrchestrator: self, db: Session, config_manager: ConfigManager, - current_user: Optional[str] = None, + current_user: str | None = None, ): self.db = db self.config_manager = config_manager self.current_user = current_user self.event_log = TranslationEventLog(db) - self._job: Optional[TranslationJob] = None + self._job: TranslationJob | None = None # region start_run [TYPE Function] # @PURPOSE: Start a new translation run for a job. @@ -71,7 +69,7 @@ class TranslationOrchestrator: self, job_id: str, is_scheduled: bool = False, - trigger_type: Optional[str] = None, + trigger_type: str | None = None, ) -> TranslationRun: with belief_scope("TranslationOrchestrator.start_run"): # Load and validate job @@ -133,7 +131,7 @@ class TranslationOrchestrator: config_hash=config_hash, dict_snapshot_hash=dict_snapshot_hash, created_by=self.current_user, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) self.db.add(run) self.db.flush() @@ -173,7 +171,7 @@ class TranslationOrchestrator: def execute_run( self, run: TranslationRun, - on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, + on_batch_progress: Callable[[str, int, int, int, int], None] | None = None, skip_insert: bool = False, ) -> TranslationRun: with belief_scope("TranslationOrchestrator.execute_run"): @@ -217,7 +215,7 @@ class TranslationOrchestrator: }) run.status = "FAILED" run.error_message = f"Translation execution failed: {e}" - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() self.event_log.log_event( job_id=job.id, @@ -246,7 +244,7 @@ class TranslationOrchestrator: # Skip insert phase if requested (e.g., for preview-only execution) if skip_insert: run.status = "COMPLETED" - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() self.event_log.log_event( job_id=job.id, @@ -270,7 +268,7 @@ class TranslationOrchestrator: run.status = "COMPLETED" if insert_result.get("error_message"): run.error_message = insert_result["error_message"] - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() # Record terminal event @@ -313,7 +311,7 @@ class TranslationOrchestrator: self, job: TranslationJob, run: TranslationRun, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("TranslationOrchestrator._generate_and_insert_sql"): # Fetch successful records records = ( @@ -620,7 +618,7 @@ class TranslationOrchestrator: else: run.status = "COMPLETED" - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() self.event_log.log_event( @@ -707,7 +705,7 @@ class TranslationOrchestrator: ) run.status = "CANCELLED" - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) self.db.flush() self.event_log.log_event( @@ -731,7 +729,7 @@ class TranslationOrchestrator: # @PURPOSE: Get run status with statistics. # @PRE: run_id exists. # @POST: Returns dict with run details. - def get_run_status(self, run_id: str) -> Dict[str, Any]: + def get_run_status(self, run_id: str) -> dict[str, Any]: with belief_scope("TranslationOrchestrator.get_run_status"): run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() if not run: @@ -780,8 +778,8 @@ class TranslationOrchestrator: run_id: str, page: int = 1, page_size: int = 50, - status_filter: Optional[str] = None, - ) -> Dict[str, Any]: + status_filter: str | None = None, + ) -> dict[str, Any]: with belief_scope("TranslationOrchestrator.get_run_records"): query = self.db.query(TranslationRecord).filter( TranslationRecord.run_id == run_id @@ -831,7 +829,7 @@ class TranslationOrchestrator: job_id: str, page: int = 1, page_size: int = 20, - ) -> Tuple[int, List[Dict[str, Any]]]: + ) -> tuple[int, list[dict[str, Any]]]: with belief_scope("TranslationOrchestrator.get_run_history"): query = self.db.query(TranslationRun).filter( TranslationRun.job_id == job_id @@ -888,6 +886,7 @@ class TranslationOrchestrator: # @PURPOSE: Compute a hash of dictionary state for snapshot comparison. def _compute_dict_snapshot_hash(self, job_id: str) -> str: import hashlib + from ...models.translate import TranslationJobDictionary dict_links = ( self.db.query(TranslationJobDictionary) diff --git a/backend/src/plugins/translate/plugin.py b/backend/src/plugins/translate/plugin.py index 5486390f..def022d8 100644 --- a/backend/src/plugins/translate/plugin.py +++ b/backend/src/plugins/translate/plugin.py @@ -5,7 +5,8 @@ # @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines). # @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains. -from typing import Dict, Any, Optional +from typing import Any + from ...core.plugin_base import PluginBase @@ -29,7 +30,7 @@ class TranslatePlugin(PluginBase): def version(self) -> str: return "1.0.0" - def get_schema(self) -> Dict[str, Any]: + def get_schema(self) -> dict[str, Any]: return { "type": "object", "properties": { @@ -52,7 +53,7 @@ class TranslatePlugin(PluginBase): "required": ["job_id"], } - async def execute(self, params: Dict[str, Any], context: Optional[Any] = None): + async def execute(self, params: dict[str, Any], context: Any | None = None): """Execute a translation job — implementation deferred to later phases.""" raise NotImplementedError("TranslatePlugin.execute not yet implemented") # #endregion TranslatePlugin diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index 3431b45a..69068478 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -15,25 +15,25 @@ # @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls. # @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. -from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.orm import Session -from datetime import datetime, timezone, timedelta -import uuid -import json import hashlib -import re +import json +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy.orm import Session -from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager +from ...core.logger import belief_scope, logger from ...core.superset_client import SupersetClient from ...models.translate import ( TranslationJob, - TranslationPreviewSession, - TranslationPreviewRecord, TranslationJobDictionary, + TranslationPreviewRecord, + TranslationPreviewSession, ) -from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt +from ...services.llm_provider import LLMProviderService from .dictionary import DictionaryManager # #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant] @@ -107,7 +107,7 @@ class TokenEstimator: # @PRE: total_tokens >= 0. # @POST: Returns estimated cost in USD. @staticmethod - def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float: + def estimate_cost(total_tokens: int, cost_per_1k: float | None = None) -> float: rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K return round((total_tokens / 1000) * rate, 6) # endregion estimate_cost @@ -127,7 +127,7 @@ class TranslationPreview: self, db: Session, config_manager: ConfigManager, - current_user: Optional[str] = None, + current_user: str | None = None, ): self.db = db self.config_manager = config_manager @@ -142,9 +142,9 @@ class TranslationPreview: self, job_id: str, sample_size: int = 10, - prompt_template: Optional[str] = None, - env_id: Optional[str] = None, - ) -> Dict[str, Any]: + prompt_template: str | None = None, + env_id: str | None = None, + ) -> dict[str, Any]: with belief_scope("TranslationPreview.preview_rows"): logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size}) @@ -189,7 +189,7 @@ class TranslationPreview: # 4. Build prompt context from rows all_source_texts = [] - row_meta: List[Dict[str, Any]] = [] + row_meta: list[dict[str, Any]] = [] for idx, row in enumerate(source_rows): translation_value = str(row.get(job.translation_column, "") or "") context_values = {} @@ -274,8 +274,8 @@ class TranslationPreview: job_id=job_id, status="ACTIVE", created_by=self.current_user, - created_at=datetime.now(timezone.utc), - expires_at=datetime.now(timezone.utc) + timedelta(hours=24), + created_at=datetime.now(UTC), + expires_at=datetime.now(UTC) + timedelta(hours=24), ) self.db.add(session) self.db.flush() @@ -313,7 +313,7 @@ class TranslationPreview: source_data=source_data, status=status, feedback=feedback, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) self.db.add(record) self.db.flush() @@ -369,9 +369,9 @@ class TranslationPreview: job_id: str, row_id: str, action: str, - translation: Optional[str] = None, - feedback: Optional[str] = None, - ) -> Dict[str, Any]: + translation: str | None = None, + feedback: str | None = None, + ) -> dict[str, Any]: with belief_scope("TranslationPreview.update_preview_row"): # Find the active session for this job session = ( @@ -434,7 +434,7 @@ class TranslationPreview: # @PRE: job_id has an ACTIVE preview session. # @POST: Session status changes to APPLIED. # @SIDE_EFFECT: Future full execution calls will check for accepted session. - def accept_preview_session(self, job_id: str) -> Dict[str, Any]: + def accept_preview_session(self, job_id: str) -> dict[str, Any]: with belief_scope("TranslationPreview.accept_preview_session"): session = ( self.db.query(TranslationPreviewSession) @@ -488,7 +488,7 @@ class TranslationPreview: # @PURPOSE: Get the latest preview session for a job with its records. # @PRE: job_id exists. # @POST: Returns session data with records or raises ValueError. - def get_preview_session(self, job_id: str) -> Dict[str, Any]: + def get_preview_session(self, job_id: str) -> dict[str, Any]: with belief_scope("TranslationPreview.get_preview_session"): session = ( self.db.query(TranslationPreviewSession) @@ -533,7 +533,7 @@ class TranslationPreview: # @PRE: job has source_datasource_id and translation_column. # @POST: Returns list of dicts with row data. # @SIDE_EFFECT: Calls Superset chart data endpoint. - def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]: + def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: str | None = None) -> list[dict[str, Any]]: with belief_scope("TranslationPreview._fetch_sample_rows"): # Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy) environments = self.config_manager.get_environments() @@ -618,7 +618,7 @@ class TranslationPreview: # @PRE: response is a dict from Superset API. # @POST: Returns list of row dicts. @staticmethod - def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: + def _extract_data_rows(response: dict[str, Any]) -> list[dict[str, Any]]: with belief_scope("TranslationPreview._extract_data_rows"): # Try various response formats result = response.get("result") @@ -756,7 +756,7 @@ class TranslationPreview: # @PRE: response_text is valid JSON with {"rows": [...]} structure. # @POST: Returns dict mapping string row_id to translation text. @staticmethod - def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: + def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]: with belief_scope("TranslationPreview._parse_llm_response"): logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}") @@ -779,7 +779,7 @@ class TranslationPreview: logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}") raise ValueError("LLM response missing 'rows' array") - translations: Dict[str, str] = {} + translations: dict[str, str] = {} for item in rows: row_id = str(item.get("row_id", "")) translation = str(item.get("translation", "")) diff --git a/backend/src/plugins/translate/scheduler.py b/backend/src/plugins/translate/scheduler.py index e60a8e00..574f23cc 100644 --- a/backend/src/plugins/translate/scheduler.py +++ b/backend/src/plugins/translate/scheduler.py @@ -13,15 +13,14 @@ # @REJECTED: Polling-based approach — event-driven APScheduler is more precise. import uuid -from datetime import datetime, timezone, timedelta -from typing import Any, Dict, List, Optional +from datetime import UTC, datetime, timedelta from sqlalchemy.orm import Session -from ...core.logger import logger, belief_scope -from ...core.cot_logger import seed_trace_id from ...core.config_manager import ConfigManager -from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun +from ...core.cot_logger import seed_trace_id +from ...core.logger import belief_scope, logger +from ...models.translate import TranslationJob, TranslationRun, TranslationSchedule from .events import TranslationEventLog @@ -29,7 +28,7 @@ from .events import TranslationEventLog # @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers. class TranslationScheduler: - def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): + def __init__(self, db: Session, config_manager: ConfigManager, current_user: str | None = None): self.db = db self.config_manager = config_manager self.current_user = current_user @@ -88,10 +87,10 @@ class TranslationScheduler: def update_schedule( self, job_id: str, - cron_expression: Optional[str] = None, - timezone_str: Optional[str] = None, - is_active: Optional[bool] = None, - execution_mode: Optional[str] = None, + cron_expression: str | None = None, + timezone_str: str | None = None, + is_active: bool | None = None, + execution_mode: str | None = None, ) -> TranslationSchedule: with belief_scope("TranslationScheduler.update_schedule"): schedule = self.db.query(TranslationSchedule).filter( @@ -108,7 +107,7 @@ class TranslationScheduler: schedule.is_active = is_active if execution_mode is not None: schedule.execution_mode = execution_mode - schedule.updated_at = datetime.now(timezone.utc) + schedule.updated_at = datetime.now(UTC) self.db.commit() self.db.refresh(schedule) @@ -167,7 +166,7 @@ class TranslationScheduler: raise ValueError(f"No schedule found for job '{job_id}'") schedule.is_active = is_active - schedule.updated_at = datetime.now(timezone.utc) + schedule.updated_at = datetime.now(UTC) self.db.commit() self.db.refresh(schedule) @@ -197,7 +196,7 @@ class TranslationScheduler: # @PURPOSE: List all active schedules. # @POST: Returns list of active TranslationSchedule rows. @staticmethod - def list_active_schedules(db: Session) -> List[TranslationSchedule]: + def list_active_schedules(db: Session) -> list[TranslationSchedule]: return ( db.query(TranslationSchedule) .filter(TranslationSchedule.is_active == True) @@ -210,10 +209,10 @@ class TranslationScheduler: # @PRE: cron_expression is valid. # @POST: Returns list of ISO datetime strings. @staticmethod - def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]: + def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> list[str]: from zoneinfo import ZoneInfo + from apscheduler.triggers.cron import CronTrigger - from apscheduler.triggers.interval import IntervalTrigger try: tz = ZoneInfo(timezone_str) @@ -274,7 +273,7 @@ def execute_scheduled_translation( # Concurrency check: max 1 pending/running run per job STALE_THRESHOLD = timedelta(hours=1) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) active_run = ( db.query(TranslationRun) @@ -337,7 +336,7 @@ def execute_scheduled_translation( .first() ) if most_recent: - age = datetime.now(timezone.utc) - most_recent.created_at + age = datetime.now(UTC) - most_recent.created_at if age > timedelta(days=90): baseline_expired = True logger.reason("Baseline expired — full translation", { @@ -388,10 +387,10 @@ def execute_scheduled_translation( # Leave schedule enabled — schedule continues on failure run.status = "FAILED" run.error_message = str(exec_err) - run.completed_at = datetime.now(timezone.utc) + run.completed_at = datetime.now(UTC) # Update schedule tracking - schedule.last_run_at = datetime.now(timezone.utc) + schedule.last_run_at = datetime.now(UTC) db.commit() logger.reflect("Scheduled translation complete", { diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index f76721a2..a94526bc 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -10,21 +10,22 @@ # @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. # @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. -from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.orm import Session -from datetime import datetime, timezone import uuid +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.orm import Session -from ...core.logger import logger from ...core.config_manager import ConfigManager +from ...core.logger import logger from ...core.superset_client import SupersetClient from ...models.translate import TranslationJob, TranslationJobDictionary from ...schemas.translate import ( - TranslateJobCreate, - TranslateJobUpdate, - TranslateJobResponse, - DatasourceColumnsResponse, DatasourceColumnResponse, + DatasourceColumnsResponse, + TranslateJobCreate, + TranslateJobResponse, + TranslateJobUpdate, ) # Supported database dialects for translation @@ -39,7 +40,7 @@ SUPPORTED_DIALECTS = { # @BRIEF Extract normalized dialect string from a Superset database record. # @PRE: database_record is a dict from Superset API. # @POST: Returns normalized dialect string or raises ValueError if unsupported. -def get_dialect_from_database(database_record: Dict[str, Any]) -> str: +def get_dialect_from_database(database_record: dict[str, Any]) -> str: """Extract and validate dialect from Superset database record.""" backend = ( database_record.get("backend") @@ -89,7 +90,7 @@ def fetch_datasource_metadata( dataset_id: int, env_id: str, config_manager: ConfigManager, -) -> Tuple[List[Dict[str, Any]], str]: +) -> tuple[list[dict[str, Any]], str]: """Fetch column metadata and database dialect for a datasource from Superset.""" # Find environment config environments = config_manager.get_environments() @@ -142,7 +143,7 @@ def fetch_datasource_metadata( # @BRIEF Identify virtual (calculated) columns from column metadata. # @PRE: columns is a list of dicts with 'is_physical' key. # @POST: Returns list of virtual column names. -def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]: +def detect_virtual_columns(columns: list[dict[str, Any]]) -> list[str]: """Return names of columns that are virtual (not physical).""" return [col["name"] for col in columns if not col.get("is_physical", True)] # #endregion detect_virtual_columns @@ -152,7 +153,7 @@ def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]: # @BRIEF Service for translation job CRUD with validation and Superset integration. class TranslateJobService: - def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): + def __init__(self, db: Session, config_manager: ConfigManager, current_user: str | None = None): self.db = db self.config_manager = config_manager self.current_user = current_user @@ -164,8 +165,8 @@ class TranslateJobService: self, page: int = 1, page_size: int = 20, - status_filter: Optional[str] = None, - ) -> Tuple[int, List[TranslationJob]]: + status_filter: str | None = None, + ) -> tuple[int, list[TranslationJob]]: query = self.db.query(TranslationJob) if status_filter: @@ -300,7 +301,7 @@ class TranslateJobService: except Exception as e: logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}") - job.updated_at = datetime.now(timezone.utc) + job.updated_at = datetime.now(UTC) # Update dictionary associations if provided if dict_ids is not None: @@ -343,7 +344,7 @@ class TranslateJobService: # @PURPOSE: Duplicate a translation job with a new name. # @PRE: job_id must exist. # @POST: Returns the duplicated TranslationJob with status DRAFT. - def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob: + def duplicate_job(self, job_id: str, new_name: str | None = None) -> TranslationJob: logger.info(f"[TranslateJobService] Duplicating job '{job_id}'") source = self.get_job(job_id) @@ -395,7 +396,7 @@ class TranslateJobService: # region get_job_dictionary_ids [TYPE Function] # @PURPOSE: Get dictionary IDs attached to a job. # @POST: Returns list of dictionary IDs. - def get_job_dictionary_ids(self, job_id: str) -> List[str]: + def get_job_dictionary_ids(self, job_id: str) -> list[str]: assocs = self.db.query(TranslationJobDictionary).filter( TranslationJobDictionary.job_id == job_id ).all() @@ -404,7 +405,7 @@ class TranslateJobService: # region fetch_available_datasources [TYPE Function] # @PURPOSE: List available Superset datasets for translation job creation. - def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list: + def fetch_available_datasources(self, env_id: str, search: str | None = None) -> list: """List Superset datasets available for translation.""" from ...core.superset_client import SupersetClient env = self.config_manager.get_environment(env_id) @@ -449,7 +450,7 @@ def _extract_dialect(backend: str) -> str: # #region job_to_response [TYPE Function] # @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids. -def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse: +def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> TranslateJobResponse: return TranslateJobResponse( id=job.id, name=job.name, diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py index 3a3a9fd5..363c7836 100644 --- a/backend/src/plugins/translate/sql_generator.py +++ b/backend/src/plugins/translate/sql_generator.py @@ -10,9 +10,10 @@ # @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case. # @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail. -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Tuple -from ...core.logger import logger, belief_scope +from datetime import UTC, datetime +from typing import Any + +from ...core.logger import belief_scope, logger # PostgreSQL/Greenplum dialects that support ON CONFLICT POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"} @@ -22,7 +23,7 @@ CLICKHOUSE_DIALECTS = {"clickhouse"} # #region _normalize_timestamp_value [C:2] [TYPE Function] [SEMANTICS translate,sql,timestamp] # @BRIEF Detect Unix timestamp strings (seconds or millis) and convert to 'YYYY-MM-DD' for Date columns. -def _normalize_timestamp_value(value: Any) -> Optional[str]: +def _normalize_timestamp_value(value: Any) -> str | None: """Detect Unix timestamp values and convert to date string. Handles: @@ -50,7 +51,7 @@ def _normalize_timestamp_value(value: Any) -> Optional[str]: return None try: - dt = datetime.fromtimestamp(ts, tz=timezone.utc) + dt = datetime.fromtimestamp(ts, tz=UTC) return dt.strftime("%Y-%m-%d") except (OSError, OverflowError, ValueError): return None @@ -79,7 +80,7 @@ def _quote_identifier(identifier: str, dialect: str) -> str: # #region _encode_sql_value [C:2] [TYPE Function] [SEMANTICS translate,sql,encoding] # @BRIEF Encode a Python value into a SQL-safe literal for INSERT VALUES, with ClickHouse timestamp normalization. -def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str: +def _encode_sql_value(value: Any, dialect: str | None = None) -> str: """Encode a Python value into a SQL-safe literal. For ClickHouse dialect, attempts to detect Unix timestamp strings @@ -92,11 +93,7 @@ def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str: # For ClickHouse: try to normalize timestamp-like values (both string and numeric) if dialect in CLICKHOUSE_DIALECTS: - if isinstance(value, str) and value: - normalized = _normalize_timestamp_value(value) - if normalized: - return f"'{normalized}'" - elif isinstance(value, (int, float)): + if (isinstance(value, str) and value) or isinstance(value, (int, float)): normalized = _normalize_timestamp_value(value) if normalized: return f"'{normalized}'" @@ -111,7 +108,7 @@ def _encode_sql_value(value: Any, dialect: Optional[str] = None) -> str: # #region _build_values_clause [C:2] [TYPE Function] [SEMANTICS translate,sql,values] # @BRIEF Build a VALUES clause for multiple rows with per-column value encoding and dialect-aware quoting. -def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect: Optional[str] = None) -> str: +def _build_values_clause(columns: list[str], rows: list[dict[str, Any]], dialect: str | None = None) -> str: """Build VALUES (...) clause for multiple rows. NOTE: columns may be quoted (e.g. '"col"' or '`col`') for the SQL column list, @@ -135,11 +132,11 @@ def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]], dialect # @RELATION DEPENDS_ON -> [_quote_identifier] # @RELATION DEPENDS_ON -> [_build_values_clause] def generate_insert_sql( - target_schema: Optional[str], + target_schema: str | None, target_table: str, - columns: List[str], - rows: List[Dict[str, Any]], - dialect: Optional[str] = None, + columns: list[str], + rows: list[dict[str, Any]], + dialect: str | None = None, ) -> str: """Generate a plain INSERT SQL.""" with belief_scope("generate_insert_sql"): @@ -167,11 +164,11 @@ def generate_insert_sql( # @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty. # @POST: Returns UPSERT SQL string or raises ValueError. def generate_upsert_sql( - target_schema: Optional[str], + target_schema: str | None, target_table: str, - columns: List[str], - key_columns: List[str], - rows: List[Dict[str, Any]], + columns: list[str], + key_columns: list[str], + rows: list[dict[str, Any]], ) -> str: """Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.""" with belief_scope("generate_upsert_sql"): @@ -225,13 +222,13 @@ class SQLGenerator: @staticmethod def generate( dialect: str, - target_schema: Optional[str], + target_schema: str | None, target_table: str, - columns: List[str], - rows: List[Dict[str, Any]], - key_columns: Optional[List[str]] = None, + columns: list[str], + rows: list[dict[str, Any]], + key_columns: list[str] | None = None, upsert_strategy: str = "MERGE", - ) -> Tuple[str, int]: + ) -> tuple[str, int]: """Generate dialect-appropriate INSERT/UPSERT SQL. Args: @@ -340,14 +337,14 @@ class SQLGenerator: @staticmethod def generate_batch( dialect: str, - target_schema: Optional[str], + target_schema: str | None, target_table: str, - columns: List[str], - rows: List[Dict[str, Any]], - key_columns: Optional[List[str]] = None, + columns: list[str], + rows: list[dict[str, Any]], + key_columns: list[str] | None = None, upsert_strategy: str = "MERGE", max_rows_per_statement: int = 500, - ) -> List[Tuple[str, int]]: + ) -> list[tuple[str, int]]: """Generate SQL in batches, splitting large row sets into multiple statements. Returns: diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index 7ae5236c..06bce766 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -9,14 +9,13 @@ # @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. # @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. -from typing import Any, Dict, List, Optional, Tuple -from datetime import datetime, timezone import json import time import uuid +from typing import Any -from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager +from ...core.logger import belief_scope, logger from ...core.superset_client import SupersetClient @@ -30,9 +29,9 @@ class SupersetSqlLabExecutor: def __init__(self, config_manager: ConfigManager, env_id: str): self.config_manager = config_manager self.env_id = env_id - self._client: Optional[SupersetClient] = None - self._database_id: Optional[int] = None - self._database_backend: Optional[str] = None + self._client: SupersetClient | None = None + self._database_id: int | None = None + self._database_backend: str | None = None # region _get_client [TYPE Function] # @PURPOSE: Lazy-initialize SupersetClient for the configured environment. @@ -60,8 +59,8 @@ class SupersetSqlLabExecutor: # @SIDE_EFFECT: Fetches databases list from Superset; optionally fetches single DB for backend. def resolve_database_id( self, - database_name: Optional[str] = None, - target_database_id: Optional[str] = None, + database_name: str | None = None, + target_database_id: str | None = None, ) -> int: with belief_scope("SupersetSqlLabExecutor.resolve_database_id"): client = self._get_client() @@ -124,7 +123,7 @@ class SupersetSqlLabExecutor: # region get_database_backend [TYPE Function] # @PURPOSE: Return the cached database backend/engine string. # @POST: Returns backend string or None if not yet resolved. - def get_database_backend(self) -> Optional[str]: + def get_database_backend(self) -> str | None: return self._database_backend # endregion get_database_backend @@ -136,9 +135,9 @@ class SupersetSqlLabExecutor: def execute_sql( self, sql: str, - database_id: Optional[int] = None, + database_id: int | None = None, run_async: bool = True, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.execute_sql"): client = self._get_client() db_id = database_id or self._database_id @@ -174,13 +173,13 @@ class SupersetSqlLabExecutor: headers={"Content-Type": "application/json"}, ) if isinstance(response, dict) and response: - logger.reason(f"SQL Lab endpoint succeeded", extra={ + logger.reason("SQL Lab endpoint succeeded", extra={ "endpoint": endpoint, }) break except Exception as ep_err: last_error = ep_err - logger.explore(f"SQL Lab endpoint failed, trying next", extra={ + logger.explore("SQL Lab endpoint failed, trying next", extra={ "error": str(ep_err), "endpoint": endpoint, }) @@ -246,7 +245,7 @@ class SupersetSqlLabExecutor: query_id: str, max_polls: int = 60, poll_interval_seconds: float = 2.0, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.poll_execution_status"): client = self._get_client() @@ -337,10 +336,10 @@ class SupersetSqlLabExecutor: def execute_and_poll( self, sql: str, - database_id: Optional[int] = None, + database_id: int | None = None, max_polls: int = 60, poll_interval_seconds: float = 2.0, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.execute_and_poll"): exec_result = self.execute_sql( sql=sql, @@ -389,7 +388,7 @@ class SupersetSqlLabExecutor: # @PRE: query_id is a valid Superset query ID. # @POST: Returns query results if available. # @SIDE_EFFECT: Makes HTTP GET to Superset. - def get_query_results(self, query_id: str) -> Dict[str, Any]: + def get_query_results(self, query_id: str) -> dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.get_query_results"): client = self._get_client() try: diff --git a/backend/src/schemas/__tests__/test_settings_and_health_schemas.py b/backend/src/schemas/__tests__/test_settings_and_health_schemas.py index f0e5d28a..6e626009 100644 --- a/backend/src/schemas/__tests__/test_settings_and_health_schemas.py +++ b/backend/src/schemas/__tests__/test_settings_and_health_schemas.py @@ -3,8 +3,11 @@ # @BRIEF Regression tests for settings and health schema contracts updated in 026 fix batch. import pytest from pydantic import ValidationError + from src.schemas.health import DashboardHealthItem from src.schemas.settings import ValidationPolicyCreate + + # #region test_validation_policy_create_accepts_structured_custom_channels [TYPE Function] # @RELATION BINDS_TO -> TestSettingsAndHealthSchemas # @BRIEF Ensure policy schema accepts structured custom channel objects with type/target fields. @@ -69,4 +72,4 @@ def test_dashboard_health_item_status_accepts_only_whitelisted_values(): last_check="2026-03-10T10:00:00", ) # #endregion test_dashboard_health_item_status_accepts_only_whitelisted_values -# #endregion TestSettingsAndHealthSchemas \ No newline at end of file +# #endregion TestSettingsAndHealthSchemas diff --git a/backend/src/schemas/auth.py b/backend/src/schemas/auth.py index e8213998..079f3679 100644 --- a/backend/src/schemas/auth.py +++ b/backend/src/schemas/auth.py @@ -6,10 +6,10 @@ # # @INVARIANT: Sensitive fields like password must not be included in response schemas. -from typing import List, Optional -from pydantic import BaseModel, EmailStr from datetime import datetime +from pydantic import BaseModel, EmailStr + # #region Token [C:1] [TYPE Class] # @BRIEF Represents a JWT access token response. @@ -24,8 +24,8 @@ class Token(BaseModel): # #region TokenData [C:1] [TYPE Class] # @BRIEF Represents the data encoded in a JWT token. class TokenData(BaseModel): - username: Optional[str] = None - scopes: List[str] = [] + username: str | None = None + scopes: list[str] = [] # #endregion TokenData @@ -34,7 +34,7 @@ class TokenData(BaseModel): # #region PermissionSchema [C:1] [TYPE Class] # @BRIEF Represents a permission in API responses. class PermissionSchema(BaseModel): - id: Optional[str] = None + id: str | None = None resource: str action: str @@ -50,8 +50,8 @@ class PermissionSchema(BaseModel): class RoleSchema(BaseModel): id: str name: str - description: Optional[str] = None - permissions: List[PermissionSchema] = [] + description: str | None = None + permissions: list[PermissionSchema] = [] class Config: from_attributes = True @@ -64,8 +64,8 @@ class RoleSchema(BaseModel): # @BRIEF Schema for creating a new role. class RoleCreate(BaseModel): name: str - description: Optional[str] = None - permissions: List[str] = [] # List of permission IDs or "resource:action" strings + description: str | None = None + permissions: list[str] = [] # List of permission IDs or "resource:action" strings # #endregion RoleCreate @@ -74,9 +74,9 @@ class RoleCreate(BaseModel): # #region RoleUpdate [TYPE Class] # @BRIEF Schema for updating an existing role. class RoleUpdate(BaseModel): - name: Optional[str] = None - description: Optional[str] = None - permissions: Optional[List[str]] = None + name: str | None = None + description: str | None = None + permissions: list[str] | None = None # #endregion RoleUpdate @@ -110,7 +110,7 @@ class ADGroupMappingCreate(BaseModel): # @BRIEF Base schema for user data. class UserBase(BaseModel): username: str - email: Optional[EmailStr] = None + email: EmailStr | None = None is_active: bool = True @@ -121,7 +121,7 @@ class UserBase(BaseModel): # @BRIEF Schema for creating a new user. class UserCreate(UserBase): password: str - roles: List[str] = [] + roles: list[str] = [] # #endregion UserCreate @@ -130,10 +130,10 @@ class UserCreate(UserBase): # #region UserUpdate [TYPE Class] # @BRIEF Schema for updating an existing user. class UserUpdate(BaseModel): - email: Optional[EmailStr] = None - password: Optional[str] = None - is_active: Optional[bool] = None - roles: Optional[List[str]] = None + email: EmailStr | None = None + password: str | None = None + is_active: bool | None = None + roles: list[str] | None = None # #endregion UserUpdate @@ -145,8 +145,8 @@ class User(UserBase): id: str auth_source: str created_at: datetime - last_login: Optional[datetime] = None - roles: List[RoleSchema] = [] + last_login: datetime | None = None + roles: list[RoleSchema] = [] class Config: from_attributes = True diff --git a/backend/src/schemas/dataset_review.py b/backend/src/schemas/dataset_review.py index 04722575..64ceb352 100644 --- a/backend/src/schemas/dataset_review.py +++ b/backend/src/schemas/dataset_review.py @@ -4,25 +4,25 @@ # @RATIONALE: Original 419-line file exceeded INV_7 (400-line module limit). Decomposed into DTO and composite sub-modules. # @REJECTED: Keeping all schemas in a single file because it exceeded the fractal limit. -from src.schemas.dataset_review_pkg._dtos import ( # noqa: F401 - SessionCollaboratorDto, - DatasetProfileDto, - ValidationFindingDto, - SemanticSourceDto, - SemanticCandidateDto, - SemanticFieldEntryDto, - ImportedFilterDto, - TemplateVariableDto, - ExecutionMappingDto, -) from src.schemas.dataset_review_pkg._composites import ( # noqa: F401 - ClarificationOptionDto, ClarificationAnswerDto, + ClarificationOptionDto, ClarificationQuestionDto, ClarificationSessionDto, CompiledPreviewDto, DatasetRunContextDto, - SessionSummary, SessionDetail, + SessionSummary, +) +from src.schemas.dataset_review_pkg._dtos import ( # noqa: F401 + DatasetProfileDto, + ExecutionMappingDto, + ImportedFilterDto, + SemanticCandidateDto, + SemanticFieldEntryDto, + SemanticSourceDto, + SessionCollaboratorDto, + TemplateVariableDto, + ValidationFindingDto, ) # #endregion DatasetReviewSchemas diff --git a/backend/src/schemas/dataset_review_pkg/_composites.py b/backend/src/schemas/dataset_review_pkg/_composites.py index 5ea0b9a9..973b435c 100644 --- a/backend/src/schemas/dataset_review_pkg/_composites.py +++ b/backend/src/schemas/dataset_review_pkg/_composites.py @@ -4,30 +4,30 @@ # @RELATION DEPENDS_ON -> [DatasetReviewSchemaDtos] from datetime import datetime -from typing import Any, List, Optional +from typing import Any from pydantic import BaseModel from src.models.dataset_review import ( - ClarificationStatus, - QuestionState, AnswerKind, - PreviewStatus, + ClarificationStatus, LaunchStatus, - SessionStatus, - SessionPhase, + PreviewStatus, + QuestionState, ReadinessState, RecommendedAction, + SessionPhase, + SessionStatus, ) from src.schemas.dataset_review_pkg._dtos import ( - SessionCollaboratorDto, DatasetProfileDto, - ValidationFindingDto, - SemanticSourceDto, - SemanticFieldEntryDto, - ImportedFilterDto, - TemplateVariableDto, ExecutionMappingDto, + ImportedFilterDto, + SemanticFieldEntryDto, + SemanticSourceDto, + SessionCollaboratorDto, + TemplateVariableDto, + ValidationFindingDto, ) @@ -54,10 +54,10 @@ class ClarificationAnswerDto(BaseModel): answer_id: str question_id: str answer_kind: AnswerKind - answer_value: Optional[str] = None + answer_value: str | None = None answered_by_user_id: str - impact_summary: Optional[str] = None - user_feedback: Optional[str] = None + impact_summary: str | None = None + user_feedback: str | None = None created_at: datetime class Config: @@ -75,13 +75,13 @@ class ClarificationQuestionDto(BaseModel): topic_ref: str question_text: str why_it_matters: str - current_guess: Optional[str] = None + current_guess: str | None = None priority: int state: QuestionState created_at: datetime updated_at: datetime - options: List[ClarificationOptionDto] = [] - answer: Optional[ClarificationAnswerDto] = None + options: list[ClarificationOptionDto] = [] + answer: ClarificationAnswerDto | None = None class Config: from_attributes = True @@ -96,14 +96,14 @@ class ClarificationSessionDto(BaseModel): clarification_session_id: str session_id: str status: ClarificationStatus - current_question_id: Optional[str] = None + current_question_id: str | None = None resolved_count: int remaining_count: int - summary_delta: Optional[str] = None + summary_delta: str | None = None started_at: datetime updated_at: datetime - completed_at: Optional[datetime] = None - questions: List[ClarificationQuestionDto] = [] + completed_at: datetime | None = None + questions: list[ClarificationQuestionDto] = [] class Config: from_attributes = True @@ -117,14 +117,14 @@ class ClarificationSessionDto(BaseModel): class CompiledPreviewDto(BaseModel): preview_id: str session_id: str - session_version: Optional[int] = None + session_version: int | None = None preview_status: PreviewStatus - compiled_sql: Optional[str] = None + compiled_sql: str | None = None preview_fingerprint: str compiled_by: str - error_code: Optional[str] = None - error_details: Optional[str] = None - compiled_at: Optional[datetime] = None + error_code: str | None = None + error_details: str | None = None + compiled_at: datetime | None = None created_at: datetime class Config: @@ -139,18 +139,18 @@ class CompiledPreviewDto(BaseModel): class DatasetRunContextDto(BaseModel): run_context_id: str session_id: str - session_version: Optional[int] = None + session_version: int | None = None dataset_ref: str environment_id: str preview_id: str sql_lab_session_ref: str effective_filters: Any template_params: Any - approved_mapping_ids: List[str] - semantic_decision_refs: List[str] - open_warning_refs: List[str] + approved_mapping_ids: list[str] + semantic_decision_refs: list[str] + open_warning_refs: list[str] launch_status: LaunchStatus - launch_error: Optional[str] = None + launch_error: str | None = None created_at: datetime class Config: @@ -169,7 +169,7 @@ class SessionSummary(BaseModel): source_kind: str source_input: str dataset_ref: str - dataset_id: Optional[int] = None + dataset_id: int | None = None version: int = 0 session_version: int = 0 readiness_state: ReadinessState @@ -191,17 +191,17 @@ class SessionSummary(BaseModel): # @BRIEF Full session detail DTO with all nested aggregates for detail views. # @RELATION INHERITS -> [SessionSummary] class SessionDetail(SessionSummary): - collaborators: List[SessionCollaboratorDto] = [] - profile: Optional[DatasetProfileDto] = None - findings: List[ValidationFindingDto] = [] - semantic_sources: List[SemanticSourceDto] = [] - semantic_fields: List[SemanticFieldEntryDto] = [] - imported_filters: List[ImportedFilterDto] = [] - template_variables: List[TemplateVariableDto] = [] - execution_mappings: List[ExecutionMappingDto] = [] - clarification_sessions: List[ClarificationSessionDto] = [] - previews: List[CompiledPreviewDto] = [] - run_contexts: List[DatasetRunContextDto] = [] + collaborators: list[SessionCollaboratorDto] = [] + profile: DatasetProfileDto | None = None + findings: list[ValidationFindingDto] = [] + semantic_sources: list[SemanticSourceDto] = [] + semantic_fields: list[SemanticFieldEntryDto] = [] + imported_filters: list[ImportedFilterDto] = [] + template_variables: list[TemplateVariableDto] = [] + execution_mappings: list[ExecutionMappingDto] = [] + clarification_sessions: list[ClarificationSessionDto] = [] + previews: list[CompiledPreviewDto] = [] + run_contexts: list[DatasetRunContextDto] = [] # #endregion SessionDetail diff --git a/backend/src/schemas/dataset_review_pkg/_dtos.py b/backend/src/schemas/dataset_review_pkg/_dtos.py index 49d09b18..64ae5fa8 100644 --- a/backend/src/schemas/dataset_review_pkg/_dtos.py +++ b/backend/src/schemas/dataset_review_pkg/_dtos.py @@ -4,36 +4,32 @@ # @RELATION DEPENDS_ON -> [DatasetReviewModels] from datetime import datetime -from typing import List, Optional, Any +from typing import Any from pydantic import BaseModel from src.models.dataset_review import ( - SessionStatus, - SessionPhase, - ReadinessState, - RecommendedAction, - SessionCollaboratorRole, + ApprovalState, BusinessSummarySource, - ConfidenceState, - FindingArea, - FindingSeverity, - ResolutionState, - SemanticSourceType, - TrustLevel, - SemanticSourceStatus, - FieldKind, - FieldProvenance, CandidateMatchType, CandidateStatus, - FilterSource, + ConfidenceState, + FieldKind, + FieldProvenance, FilterConfidenceState, FilterRecoveryStatus, - VariableKind, - MappingStatus, + FilterSource, + FindingArea, + FindingSeverity, MappingMethod, + MappingStatus, MappingWarningLevel, - ApprovalState, + ResolutionState, + SemanticSourceStatus, + SemanticSourceType, + SessionCollaboratorRole, + TrustLevel, + VariableKind, ) @@ -57,14 +53,14 @@ class DatasetProfileDto(BaseModel): profile_id: str session_id: str dataset_name: str - schema_name: Optional[str] = None - database_name: Optional[str] = None + schema_name: str | None = None + database_name: str | None = None business_summary: str business_summary_source: BusinessSummarySource - description: Optional[str] = None - dataset_type: Optional[str] = None + description: str | None = None + dataset_type: str | None = None is_sqllab_view: bool - completeness_score: Optional[float] = None + completeness_score: float | None = None confidence_state: ConfidenceState has_blocking_findings: bool has_warning_findings: bool @@ -90,10 +86,10 @@ class ValidationFindingDto(BaseModel): title: str message: str resolution_state: ResolutionState - resolution_note: Optional[str] = None - caused_by_ref: Optional[str] = None + resolution_note: str | None = None + caused_by_ref: str | None = None created_at: datetime - resolved_at: Optional[datetime] = None + resolved_at: datetime | None = None class Config: from_attributes = True @@ -112,7 +108,7 @@ class SemanticSourceDto(BaseModel): source_version: str display_name: str trust_level: TrustLevel - schema_overlap_score: Optional[float] = None + schema_overlap_score: float | None = None status: SemanticSourceStatus created_at: datetime @@ -128,13 +124,13 @@ class SemanticSourceDto(BaseModel): class SemanticCandidateDto(BaseModel): candidate_id: str field_id: str - source_id: Optional[str] = None + source_id: str | None = None candidate_rank: int match_type: CandidateMatchType confidence_score: float - proposed_verbose_name: Optional[str] = None - proposed_description: Optional[str] = None - proposed_display_format: Optional[str] = None + proposed_verbose_name: str | None = None + proposed_description: str | None = None + proposed_display_format: str | None = None status: CandidateStatus created_at: datetime @@ -150,24 +146,24 @@ class SemanticCandidateDto(BaseModel): class SemanticFieldEntryDto(BaseModel): field_id: str session_id: str - session_version: Optional[int] = None + session_version: int | None = None field_name: str field_kind: FieldKind - verbose_name: Optional[str] = None - description: Optional[str] = None - display_format: Optional[str] = None + verbose_name: str | None = None + description: str | None = None + display_format: str | None = None provenance: FieldProvenance - source_id: Optional[str] = None - source_version: Optional[str] = None - confidence_rank: Optional[int] = None + source_id: str | None = None + source_version: str | None = None + confidence_rank: int | None = None is_locked: bool has_conflict: bool needs_review: bool last_changed_by: str - user_feedback: Optional[str] = None + user_feedback: str | None = None created_at: datetime updated_at: datetime - candidates: List[SemanticCandidateDto] = [] + candidates: list[SemanticCandidateDto] = [] class Config: from_attributes = True @@ -182,15 +178,15 @@ class ImportedFilterDto(BaseModel): filter_id: str session_id: str filter_name: str - display_name: Optional[str] = None + display_name: str | None = None raw_value: Any raw_value_masked: bool = False - normalized_value: Optional[Any] = None + normalized_value: Any | None = None source: FilterSource confidence_state: FilterConfidenceState requires_confirmation: bool recovery_status: FilterRecoveryStatus - notes: Optional[str] = None + notes: str | None = None created_at: datetime updated_at: datetime @@ -210,7 +206,7 @@ class TemplateVariableDto(BaseModel): expression_source: str variable_kind: VariableKind is_required: bool - default_value: Optional[Any] = None + default_value: Any | None = None mapping_status: MappingStatus created_at: datetime updated_at: datetime @@ -227,18 +223,18 @@ class TemplateVariableDto(BaseModel): class ExecutionMappingDto(BaseModel): mapping_id: str session_id: str - session_version: Optional[int] = None + session_version: int | None = None filter_id: str variable_id: str mapping_method: MappingMethod raw_input_value: Any - effective_value: Optional[Any] = None - transformation_note: Optional[str] = None - warning_level: Optional[MappingWarningLevel] = None + effective_value: Any | None = None + transformation_note: str | None = None + warning_level: MappingWarningLevel | None = None requires_explicit_approval: bool approval_state: ApprovalState - approved_by_user_id: Optional[str] = None - approved_at: Optional[datetime] = None + approved_by_user_id: str | None = None + approved_at: datetime | None = None created_at: datetime updated_at: datetime diff --git a/backend/src/schemas/health.py b/backend/src/schemas/health.py index daff165d..13ae705c 100644 --- a/backend/src/schemas/health.py +++ b/backend/src/schemas/health.py @@ -3,23 +3,23 @@ # @LAYER: Domain # @RELATION DEPENDS_ON -> pydantic -from pydantic import BaseModel, Field -from typing import List, Optional from datetime import datetime +from pydantic import BaseModel, Field + # #region DashboardHealthItem [TYPE Class] # @BRIEF Represents the latest health status of a single dashboard. class DashboardHealthItem(BaseModel): - record_id: Optional[str] = None + record_id: str | None = None dashboard_id: str - dashboard_slug: Optional[str] = None - dashboard_title: Optional[str] = None + dashboard_slug: str | None = None + dashboard_title: str | None = None environment_id: str status: str = Field(..., pattern="^(PASS|WARN|FAIL|UNKNOWN)$") last_check: datetime - task_id: Optional[str] = None - summary: Optional[str] = None + task_id: str | None = None + summary: str | None = None # #endregion DashboardHealthItem @@ -28,7 +28,7 @@ class DashboardHealthItem(BaseModel): # #region HealthSummaryResponse [TYPE Class] # @BRIEF Aggregated health summary for all dashboards. class HealthSummaryResponse(BaseModel): - items: List[DashboardHealthItem] + items: list[DashboardHealthItem] pass_count: int warn_count: int fail_count: int diff --git a/backend/src/schemas/profile.py b/backend/src/schemas/profile.py index 036b5939..ec89dd35 100644 --- a/backend/src/schemas/profile.py +++ b/backend/src/schemas/profile.py @@ -7,7 +7,8 @@ # @INVARIANT: Schema shapes stay stable for profile UI states and backend preference contracts. from datetime import datetime -from typing import List, Literal, Optional +from typing import Literal + from pydantic import BaseModel, Field @@ -27,11 +28,11 @@ class ProfilePermissionState(BaseModel): # @RELATION DEPENDS_ON -> ProfileSchemas class ProfileSecuritySummary(BaseModel): read_only: bool = True - auth_source: Optional[str] = None - current_role: Optional[str] = None - role_source: Optional[str] = None - roles: List[str] = Field(default_factory=list) - permissions: List[ProfilePermissionState] = Field(default_factory=list) + auth_source: str | None = None + current_role: str | None = None + role_source: str | None = None + roles: list[str] = Field(default_factory=list) + permissions: list[ProfilePermissionState] = Field(default_factory=list) # #endregion ProfileSecuritySummary @@ -42,22 +43,22 @@ class ProfileSecuritySummary(BaseModel): # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreference(BaseModel): user_id: str - superset_username: Optional[str] = None - superset_username_normalized: Optional[str] = None + superset_username: str | None = None + superset_username_normalized: str | None = None show_only_my_dashboards: bool = False show_only_slug_dashboards: bool = True - git_username: Optional[str] = None - git_email: Optional[str] = None + git_username: str | None = None + git_email: str | None = None has_git_personal_access_token: bool = False - git_personal_access_token_masked: Optional[str] = None + git_personal_access_token_masked: str | None = None start_page: Literal["dashboards", "datasets", "reports"] = "dashboards" auto_open_task_drawer: bool = True dashboards_table_density: Literal["compact", "comfortable"] = "comfortable" - telegram_id: Optional[str] = None - email_address: Optional[str] = None + telegram_id: str | None = None + email_address: str | None = None notify_on_fail: bool = True created_at: datetime @@ -74,55 +75,53 @@ class ProfilePreference(BaseModel): # @BRIEF Request payload for updating current user's profile settings. # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreferenceUpdateRequest(BaseModel): - superset_username: Optional[str] = Field( + superset_username: str | None = Field( default=None, description="Apache Superset username bound to current user profile.", ) - show_only_my_dashboards: Optional[bool] = Field( + show_only_my_dashboards: bool | None = Field( default=None, description='When true, "/dashboards" can auto-apply profile filter in main context.', ) - show_only_slug_dashboards: Optional[bool] = Field( + show_only_slug_dashboards: bool | None = Field( default=None, description='When true, "/dashboards" hides dashboards without slug by default.', ) - git_username: Optional[str] = Field( + git_username: str | None = Field( default=None, description="Git author username used for commit signature.", ) - git_email: Optional[str] = Field( + git_email: str | None = Field( default=None, description="Git author email used for commit signature.", ) - git_personal_access_token: Optional[str] = Field( + git_personal_access_token: str | None = Field( default=None, description="Personal Access Token value. Empty string clears existing token.", ) - start_page: Optional[ - Literal["dashboards", "datasets", "reports", "reports-logs"] - ] = Field( + start_page: Literal["dashboards", "datasets", "reports", "reports-logs"] | None = Field( default=None, description="Preferred start page after login.", ) - auto_open_task_drawer: Optional[bool] = Field( + auto_open_task_drawer: bool | None = Field( default=None, description="Auto-open task drawer when long-running tasks start.", ) - dashboards_table_density: Optional[Literal["compact", "comfortable", "free"]] = ( + dashboards_table_density: Literal["compact", "comfortable", "free"] | None = ( Field( default=None, description="Preferred table density for dashboard listings.", ) ) - telegram_id: Optional[str] = Field( + telegram_id: str | None = Field( default=None, description="Telegram ID for notifications.", ) - email_address: Optional[str] = Field( + email_address: str | None = Field( default=None, description="Email address for notifications (overrides system email).", ) - notify_on_fail: Optional[bool] = Field( + notify_on_fail: bool | None = Field( default=None, description="Whether to send notifications on validation failure.", ) @@ -136,8 +135,8 @@ class ProfilePreferenceUpdateRequest(BaseModel): # @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreferenceResponse(BaseModel): status: Literal["success", "error"] = "success" - message: Optional[str] = None - validation_errors: List[str] = Field(default_factory=list) + message: str | None = None + validation_errors: list[str] = Field(default_factory=list) preference: ProfilePreference security: ProfileSecuritySummary = Field(default_factory=ProfileSecuritySummary) @@ -150,7 +149,7 @@ class ProfilePreferenceResponse(BaseModel): # @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountLookupRequest(BaseModel): environment_id: str - search: Optional[str] = None + search: str | None = None page_index: int = Field(default=0, ge=0) page_size: int = Field(default=20, ge=1, le=100) sort_column: str = Field(default="username") @@ -166,9 +165,9 @@ class SupersetAccountLookupRequest(BaseModel): class SupersetAccountCandidate(BaseModel): environment_id: str username: str - display_name: Optional[str] = None - email: Optional[str] = None - is_active: Optional[bool] = None + display_name: str | None = None + email: str | None = None + is_active: bool | None = None # #endregion SupersetAccountCandidate @@ -183,8 +182,8 @@ class SupersetAccountLookupResponse(BaseModel): page_index: int = Field(ge=0) page_size: int = Field(ge=1, le=100) total: int = Field(ge=0) - warning: Optional[str] = None - items: List[SupersetAccountCandidate] = Field(default_factory=list) + warning: str | None = None + items: list[SupersetAccountCandidate] = Field(default_factory=list) # #endregion SupersetAccountLookupResponse diff --git a/backend/src/schemas/settings.py b/backend/src/schemas/settings.py index 3d3f7dad..2c135cee 100644 --- a/backend/src/schemas/settings.py +++ b/backend/src/schemas/settings.py @@ -3,10 +3,10 @@ # @LAYER: Domain # @RELATION DEPENDS_ON -> pydantic -from pydantic import BaseModel, Field -from typing import List, Optional from datetime import datetime, time +from pydantic import BaseModel, Field + # #region NotificationChannel [TYPE Class] # @BRIEF Structured notification channel definition for policy-level custom routing. @@ -28,10 +28,10 @@ class ValidationPolicyBase(BaseModel): name: str = Field(..., description="Name of the policy") environment_id: str = Field(..., description="Target Superset environment ID") is_active: bool = Field(True, description="Whether the policy is currently active") - dashboard_ids: List[str] = Field( + dashboard_ids: list[str] = Field( ..., description="List of dashboard IDs to validate" ) - schedule_days: List[int] = Field( + schedule_days: list[int] = Field( ..., description="Days of the week (0-6, 0=Sunday) to run" ) window_start: time = Field(..., description="Start of the execution window") @@ -39,7 +39,7 @@ class ValidationPolicyBase(BaseModel): notify_owners: bool = Field( True, description="Whether to notify dashboard owners on failure" ) - custom_channels: Optional[List[NotificationChannel]] = Field( + custom_channels: list[NotificationChannel] | None = Field( None, description="List of additional structured notification channels", ) @@ -64,16 +64,16 @@ class ValidationPolicyCreate(ValidationPolicyBase): # #region ValidationPolicyUpdate [TYPE Class] # @BRIEF Schema for updating an existing validation policy. class ValidationPolicyUpdate(BaseModel): - name: Optional[str] = None - environment_id: Optional[str] = None - is_active: Optional[bool] = None - dashboard_ids: Optional[List[str]] = None - schedule_days: Optional[List[int]] = None - window_start: Optional[time] = None - window_end: Optional[time] = None - notify_owners: Optional[bool] = None - custom_channels: Optional[List[NotificationChannel]] = None - alert_condition: Optional[str] = None + name: str | None = None + environment_id: str | None = None + is_active: bool | None = None + dashboard_ids: list[str] | None = None + schedule_days: list[int] | None = None + window_start: time | None = None + window_end: time | None = None + notify_owners: bool | None = None + custom_channels: list[NotificationChannel] | None = None + alert_condition: str | None = None # #endregion ValidationPolicyUpdate @@ -98,15 +98,15 @@ class ValidationPolicyResponse(ValidationPolicyBase): class TranslationScheduleItem(BaseModel): schedule_id: str job_id: str - job_name: Optional[str] = None + job_name: str | None = None cron_expression: str timezone: str is_active: bool execution_mode: str = "full" - last_run_at: Optional[datetime] = None - next_run_at: Optional[datetime] = None - created_by: Optional[str] = None - created_at: Optional[datetime] = None + last_run_at: datetime | None = None + next_run_at: datetime | None = None + created_by: str | None = None + created_at: datetime | None = None class Config: from_attributes = True diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index ac9b12cf..6b16c88e 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -4,63 +4,63 @@ # @RELATION DEPENDS_ON -> pydantic from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any + from pydantic import BaseModel, Field -import json # #region TranslateJobCreate [TYPE Class] # @BRIEF Schema for creating a new translation job. class TranslateJobCreate(BaseModel): name: str - description: Optional[str] = None + description: str | None = None source_dialect: str = Field(..., description="Source database dialect (e.g. postgresql, clickhouse)") target_dialect: str = Field(..., description="Target database dialect (e.g. postgresql, clickhouse)") - database_dialect: Optional[str] = Field(None, description="Detected dialect from Superset connection at save time") - source_datasource_id: Optional[str] = Field(None, description="Superset datasource ID") - source_table: Optional[str] = Field(None, description="Source table name") - target_schema: Optional[str] = Field(None, description="Target table schema") - target_table: Optional[str] = Field(None, description="Target table name") - source_key_cols: Optional[List[str]] = Field(default_factory=list, description="Source key column names") - target_key_cols: Optional[List[str]] = Field(default_factory=list, description="Target key column names") - translation_column: Optional[str] = Field(None, description="Source column to translate") - target_column: Optional[str] = Field(None, description="Target column for translated output (defaults to translation_column)") - context_columns: Optional[List[str]] = Field(default_factory=list, description="Context column names") - target_language: Optional[str] = Field(None, description="Target language code") - provider_id: Optional[str] = Field(None, description="LLM provider ID") + database_dialect: str | None = Field(None, description="Detected dialect from Superset connection at save time") + source_datasource_id: str | None = Field(None, description="Superset datasource ID") + source_table: str | None = Field(None, description="Source table name") + target_schema: str | None = Field(None, description="Target table schema") + target_table: str | None = Field(None, description="Target table name") + source_key_cols: list[str] | None = Field(default_factory=list, description="Source key column names") + target_key_cols: list[str] | None = Field(default_factory=list, description="Target key column names") + translation_column: str | None = Field(None, description="Source column to translate") + target_column: str | None = Field(None, description="Target column for translated output (defaults to translation_column)") + context_columns: list[str] | None = Field(default_factory=list, description="Context column names") + target_language: str | None = Field(None, description="Target language code") + provider_id: str | None = Field(None, description="LLM provider ID") batch_size: int = Field(50, description="Records per batch") upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") - dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs") - environment_id: Optional[str] = Field(None, description="Superset environment ID") - target_database_id: Optional[str] = Field(None, description="Superset database ID for SQL Lab insert target") + dictionary_ids: list[str] | None = Field(default_factory=list, description="Associated terminology dictionary IDs") + environment_id: str | None = Field(None, description="Superset environment ID") + target_database_id: str | None = Field(None, description="Superset database ID for SQL Lab insert target") # #endregion TranslateJobCreate # #region TranslateJobUpdate [TYPE Class] # @BRIEF Schema for updating an existing translation job. class TranslateJobUpdate(BaseModel): - name: Optional[str] = None - description: Optional[str] = None - source_dialect: Optional[str] = None - target_dialect: Optional[str] = None - database_dialect: Optional[str] = None - source_datasource_id: Optional[str] = None - source_table: Optional[str] = None - target_schema: Optional[str] = None - target_table: Optional[str] = None - source_key_cols: Optional[List[str]] = None - target_key_cols: Optional[List[str]] = None - translation_column: Optional[str] = None - target_column: Optional[str] = None - context_columns: Optional[List[str]] = None - target_language: Optional[str] = None - provider_id: Optional[str] = None - batch_size: Optional[int] = None - upsert_strategy: Optional[str] = None - status: Optional[str] = None - dictionary_ids: Optional[List[str]] = None - environment_id: Optional[str] = None - target_database_id: Optional[str] = None + name: str | None = None + description: str | None = None + source_dialect: str | None = None + target_dialect: str | None = None + database_dialect: str | None = None + source_datasource_id: str | None = None + source_table: str | None = None + target_schema: str | None = None + target_table: str | None = None + source_key_cols: list[str] | None = None + target_key_cols: list[str] | None = None + translation_column: str | None = None + target_column: str | None = None + context_columns: list[str] | None = None + target_language: str | None = None + provider_id: str | None = None + batch_size: int | None = None + upsert_strategy: str | None = None + status: str | None = None + dictionary_ids: list[str] | None = None + environment_id: str | None = None + target_database_id: str | None = None # #endregion TranslateJobUpdate @@ -69,30 +69,30 @@ class TranslateJobUpdate(BaseModel): class TranslateJobResponse(BaseModel): id: str name: str - description: Optional[str] = None + description: str | None = None source_dialect: str target_dialect: str - database_dialect: Optional[str] = None - source_datasource_id: Optional[str] = None - source_table: Optional[str] = None - target_schema: Optional[str] = None - target_table: Optional[str] = None - source_key_cols: Optional[List[str]] = None - target_key_cols: Optional[List[str]] = None - translation_column: Optional[str] = None - target_column: Optional[str] = None - context_columns: Optional[List[str]] = None - target_language: Optional[str] = None - provider_id: Optional[str] = None + database_dialect: str | None = None + source_datasource_id: str | None = None + source_table: str | None = None + target_schema: str | None = None + target_table: str | None = None + source_key_cols: list[str] | None = None + target_key_cols: list[str] | None = None + translation_column: str | None = None + target_column: str | None = None + context_columns: list[str] | None = None + target_language: str | None = None + provider_id: str | None = None batch_size: int = 50 upsert_strategy: str = "MERGE" status: str - created_by: Optional[str] = None + created_by: str | None = None created_at: datetime - updated_at: Optional[datetime] = None - dictionary_ids: Optional[List[str]] = None - environment_id: Optional[str] = None - target_database_id: Optional[str] = None + updated_at: datetime | None = None + dictionary_ids: list[str] | None = None + environment_id: str | None = None + target_database_id: str | None = None class Config: from_attributes = True @@ -103,10 +103,10 @@ class TranslateJobResponse(BaseModel): # @BRIEF Schema for datasource column metadata response. class DatasourceColumnResponse(BaseModel): name: str - type: Optional[str] = None + type: str | None = None is_physical: bool = True is_dttm: bool = False - description: Optional[str] = None + description: str | None = None # #endregion DatasourceColumnResponse @@ -114,10 +114,10 @@ class DatasourceColumnResponse(BaseModel): # @BRIEF Schema for datasource columns endpoint response. class DatasourceColumnsResponse(BaseModel): datasource_id: int - datasource_name: Optional[str] = None - schema_name: Optional[str] = None + datasource_name: str | None = None + schema_name: str | None = None database_dialect: str - columns: List[DatasourceColumnResponse] = [] + columns: list[DatasourceColumnResponse] = [] class Config: from_attributes = True @@ -140,7 +140,7 @@ class DuplicateJobResponse(BaseModel): # @BRIEF Schema for creating a new terminology dictionary. class DictionaryCreate(BaseModel): name: str - description: Optional[str] = None + description: str | None = None source_dialect: str target_dialect: str is_active: bool = True @@ -151,7 +151,7 @@ class DictionaryCreate(BaseModel): # @BRIEF Schema for importing entries into a terminology dictionary. class DictionaryImport(BaseModel): content: str = Field(..., description="CSV or TSV content as raw string") - delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") + delimiter: str | None = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'") preview_only: bool = Field(False, description="If true, return preview without applying") # #endregion DictionaryImport @@ -162,14 +162,14 @@ class DictionaryImport(BaseModel): class DictionaryResponse(BaseModel): id: str name: str - description: Optional[str] = None + description: str | None = None source_dialect: str target_dialect: str is_active: bool - created_by: Optional[str] = None + created_by: str | None = None created_at: datetime - updated_at: Optional[datetime] = None - entry_count: Optional[int] = None + updated_at: datetime | None = None + entry_count: int | None = None class Config: from_attributes = True @@ -181,7 +181,7 @@ class DictionaryResponse(BaseModel): class DictionaryEntryCreate(BaseModel): source_term: str = Field(..., description="Source term to translate") target_term: str = Field(..., description="Target/translated term") - context_notes: Optional[str] = Field(None, description="Optional context notes") + context_notes: str | None = Field(None, description="Optional context notes") class Config: from_attributes = True @@ -196,9 +196,9 @@ class DictionaryEntryResponse(BaseModel): source_term: str source_term_normalized: str target_term: str - context_notes: Optional[str] = None + context_notes: str | None = None created_at: datetime - updated_at: Optional[datetime] = None + updated_at: datetime | None = None class Config: from_attributes = True @@ -212,8 +212,8 @@ class DictionaryImportResult(BaseModel): created: int = 0 updated: int = 0 skipped: int = 0 - errors: List[Dict[str, Any]] = Field(default_factory=list) - preview: List[Dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags") + errors: list[dict[str, Any]] = Field(default_factory=list) + preview: list[dict[str, Any]] = Field(default_factory=list, description="Preview rows with conflict flags") # #endregion DictionaryImportResult @@ -221,8 +221,8 @@ class DictionaryImportResult(BaseModel): # @BRIEF Schema for triggering a translation preview. class PreviewRequest(BaseModel): sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview") - prompt_template: Optional[str] = Field(None, description="Optional custom prompt template") - env_id: Optional[str] = Field(None, description="Superset environment ID for preview data fetch") + prompt_template: str | None = Field(None, description="Optional custom prompt template") + env_id: str | None = Field(None, description="Superset environment ID for preview data fetch") # #endregion PreviewRequest @@ -230,8 +230,8 @@ class PreviewRequest(BaseModel): # @BRIEF Schema for approving/editing/rejecting a preview row. class PreviewRowUpdate(BaseModel): action: str = Field(..., description="'approve', 'reject', or 'edit'") - translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)") - feedback: Optional[str] = Field(None, description="Optional feedback/comment") + translation: str | None = Field(None, description="Edited translation (required for 'edit' action)") + feedback: str | None = Field(None, description="Optional feedback/comment") # #endregion PreviewRowUpdate @@ -241,10 +241,10 @@ class PreviewAcceptResponse(BaseModel): id: str job_id: str status: str - created_by: Optional[str] = None + created_by: str | None = None created_at: datetime - expires_at: Optional[datetime] = None - records: List['PreviewRow'] = [] + expires_at: datetime | None = None + records: list['PreviewRow'] = [] class Config: from_attributes = True @@ -271,16 +271,16 @@ class TermCorrectionSubmit(BaseModel): source_term: str incorrect_target_term: str corrected_target_term: str - dictionary_id: Optional[str] = Field(None, description="Target dictionary ID (language-filtered)") - origin_run_id: Optional[str] = Field(None, description="Run ID from which this correction originated") - origin_row_key: Optional[str] = Field(None, description="Row key within the run") + dictionary_id: str | None = Field(None, description="Target dictionary ID (language-filtered)") + origin_run_id: str | None = Field(None, description="Run ID from which this correction originated") + origin_row_key: str | None = Field(None, description="Row key within the run") # #endregion TermCorrectionSubmit # #region TermCorrectionBulkSubmit [TYPE Class] # @BRIEF Schema for submitting multiple term corrections atomically. class TermCorrectionBulkSubmit(BaseModel): - corrections: List[TermCorrectionSubmit] + corrections: list[TermCorrectionSubmit] dictionary_id: str = Field(..., description="Target dictionary ID for all corrections") # #endregion TermCorrectionBulkSubmit @@ -298,12 +298,12 @@ class CorrectionConflictResult(BaseModel): # #region CorrectionSubmitResponse [TYPE Class] # @BRIEF Schema for correction submission response. class CorrectionSubmitResponse(BaseModel): - entry_id: Optional[str] = None + entry_id: str | None = None action: str # "created", "updated", "conflict_detected", "skipped" source_term: str target_term: str - conflict: Optional[CorrectionConflictResult] = None - message: Optional[str] = None + conflict: CorrectionConflictResult | None = None + message: str | None = None # #endregion CorrectionSubmitResponse @@ -326,11 +326,11 @@ class ScheduleResponse(BaseModel): timezone: str is_active: bool execution_mode: str = "full" - last_run_at: Optional[datetime] = None - next_run_at: Optional[datetime] = None - created_by: Optional[str] = None + last_run_at: datetime | None = None + next_run_at: datetime | None = None + created_by: str | None = None created_at: datetime - updated_at: Optional[datetime] = None + updated_at: datetime | None = None class Config: from_attributes = True @@ -343,7 +343,7 @@ class NextExecutionResponse(BaseModel): job_id: str cron_expression: str timezone: str - next_executions: List[str] = [] + next_executions: list[str] = [] # #endregion NextExecutionResponse @@ -353,21 +353,21 @@ class TranslationRunResponse(BaseModel): id: str job_id: str status: str - trigger_type: Optional[str] = None - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - error_message: Optional[str] = None + trigger_type: str | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + error_message: str | None = None total_records: int = 0 successful_records: int = 0 failed_records: int = 0 skipped_records: int = 0 - insert_status: Optional[str] = None - superset_execution_id: Optional[str] = None - config_snapshot: Optional[Dict[str, Any]] = None - key_hash: Optional[str] = None - config_hash: Optional[str] = None - dict_snapshot_hash: Optional[str] = None - created_by: Optional[str] = None + insert_status: str | None = None + superset_execution_id: str | None = None + config_snapshot: dict[str, Any] | None = None + key_hash: str | None = None + config_hash: str | None = None + dict_snapshot_hash: str | None = None + created_by: str | None = None created_at: datetime class Config: @@ -379,9 +379,9 @@ class TranslationRunResponse(BaseModel): # @BRIEF Schema for detailed run response including records, events, and config snapshot. class RunDetailResponse(BaseModel): run: TranslationRunResponse - records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records") - events: List[Dict[str, Any]] = Field(default_factory=list, description="Run events") - event_invariants: Optional[Dict[str, Any]] = None + records: list[dict[str, Any]] = Field(default_factory=list, description="Paginated translation records") + events: list[dict[str, Any]] = Field(default_factory=list, description="Run events") + event_invariants: dict[str, Any] | None = None batch_count: int = 0 # #endregion RunDetailResponse @@ -389,12 +389,12 @@ class RunDetailResponse(BaseModel): # #region RunHistoryFilter [TYPE Class] # @BRIEF Schema for filtering run history. class RunHistoryFilter(BaseModel): - job_id: Optional[str] = None - status: Optional[str] = None - trigger_type: Optional[str] = None - created_by: Optional[str] = None - date_from: Optional[datetime] = None - date_to: Optional[datetime] = None + job_id: str | None = None + status: str | None = None + trigger_type: str | None = None + created_by: str | None = None + date_from: datetime | None = None + date_to: datetime | None = None page: int = 1 page_size: int = 20 # #endregion RunHistoryFilter @@ -403,7 +403,7 @@ class RunHistoryFilter(BaseModel): # #region RunListResponse [TYPE Class] # @BRIEF Schema for paginated run list response. class RunListResponse(BaseModel): - items: List[TranslationRunResponse] = [] + items: list[TranslationRunResponse] = [] total: int = 0 page: int = 1 page_size: int = 20 @@ -422,11 +422,11 @@ class AggregatedMetricsResponse(BaseModel): successful_records: int = 0 failed_records: int = 0 skipped_records: int = 0 - cumulative_tokens: Optional[int] = None - cumulative_cost: Optional[float] = None - avg_duration_ms: Optional[float] = None - last_run_at: Optional[datetime] = None - next_scheduled_run: Optional[datetime] = None + cumulative_tokens: int | None = None + cumulative_cost: float | None = None + avg_duration_ms: float | None = None + last_run_at: datetime | None = None + next_scheduled_run: datetime | None = None # #endregion AggregatedMetricsResponse @@ -434,13 +434,13 @@ class AggregatedMetricsResponse(BaseModel): # @BRIEF A single row in a translation preview showing original and translated content side by side. class PreviewRow(BaseModel): id: str - source_sql: Optional[str] = None - target_sql: Optional[str] = None - source_object_type: Optional[str] = None - source_object_id: Optional[str] = None - source_object_name: Optional[str] = None + source_sql: str | None = None + target_sql: str | None = None + source_object_type: str | None = None + source_object_id: str | None = None + source_object_name: str | None = None status: str = "PENDING" - feedback: Optional[str] = None + feedback: str | None = None # #endregion PreviewRow @@ -449,12 +449,12 @@ class PreviewRow(BaseModel): class TranslationPreviewResponse(BaseModel): id: str job_id: str - run_id: Optional[str] = None + run_id: str | None = None status: str - created_by: Optional[str] = None + created_by: str | None = None created_at: datetime - expires_at: Optional[datetime] = None - records: List[PreviewRow] = [] + expires_at: datetime | None = None + records: list[PreviewRow] = [] class Config: from_attributes = True @@ -471,8 +471,8 @@ class TranslationBatchResponse(BaseModel): total_records: int = 0 successful_records: int = 0 failed_records: int = 0 - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None + started_at: datetime | None = None + completed_at: datetime | None = None created_at: datetime class Config: @@ -491,10 +491,10 @@ class MetricsResponse(BaseModel): successful_records: int = 0 failed_records: int = 0 skipped_records: int = 0 - avg_duration_ms: Optional[int] = None - p50_duration_ms: Optional[int] = None - p95_duration_ms: Optional[int] = None - p99_duration_ms: Optional[int] = None + avg_duration_ms: int | None = None + p50_duration_ms: int | None = None + p95_duration_ms: int | None = None + p99_duration_ms: int | None = None class Config: from_attributes = True diff --git a/backend/src/scripts/clean_release_cli.py b/backend/src/scripts/clean_release_cli.py index 5eb846b5..286f4d16 100644 --- a/backend/src/scripts/clean_release_cli.py +++ b/backend/src/scripts/clean_release_cli.py @@ -7,8 +7,8 @@ from __future__ import annotations import argparse import json -from datetime import date, datetime, timezone -from typing import Any, Dict, List, Optional +from datetime import UTC, date, datetime +from typing import Any from ..core.cot_logger import seed_trace_id from ..models.clean_release import CandidateArtifact, ReleaseCandidate @@ -119,7 +119,7 @@ def run_candidate_register(args: argparse.Namespace) -> int: version=args.version, source_snapshot_ref=args.source_snapshot_ref, created_by=args.created_by, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.DRAFT.value, ) repository.save_candidate(candidate) @@ -216,7 +216,7 @@ def run_compliance_run(args: argparse.Namespace) -> int: requested_by=args.actor, manifest_id=args.manifest_id, ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: print(json.dumps({"status": "error", "message": str(exc)})) return 2 @@ -272,7 +272,7 @@ def run_compliance_status(args: argparse.Namespace) -> int: # @BRIEF Serialize domain models for CLI JSON output across SQLAlchemy/Pydantic variants. # @PRE: value is serializable model or primitive object. # @POST: Returns dictionary payload without mutating value. -def _to_payload(value: Any) -> Dict[str, Any]: +def _to_payload(value: Any) -> dict[str, Any]: def _normalize(raw: Any) -> Any: if isinstance(raw, datetime): return raw.isoformat() @@ -366,7 +366,7 @@ def run_approve(args: argparse.Namespace) -> int: decided_by=args.actor, comment=args.comment, ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: print(json.dumps({"status": "error", "message": str(exc)})) return 2 @@ -397,7 +397,7 @@ def run_reject(args: argparse.Namespace) -> int: decided_by=args.actor, comment=args.comment, ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: print(json.dumps({"status": "error", "message": str(exc)})) return 2 @@ -429,7 +429,7 @@ def run_publish(args: argparse.Namespace) -> int: target_channel=args.target_channel, publication_ref=args.publication_ref, ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: print(json.dumps({"status": "error", "message": str(exc)})) return 2 @@ -455,7 +455,7 @@ def run_revoke(args: argparse.Namespace) -> int: revoked_by=args.actor, comment=args.comment, ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: print(json.dumps({"status": "error", "message": str(exc)})) return 2 @@ -468,7 +468,7 @@ def run_revoke(args: argparse.Namespace) -> int: # #region main [TYPE Function] # @BRIEF CLI entrypoint for clean release commands. -def main(argv: Optional[List[str]] = None) -> int: +def main(argv: list[str] | None = None) -> int: seed_trace_id() parser = build_parser() args = parser.parse_args(argv) diff --git a/backend/src/scripts/clean_release_tui.py b/backend/src/scripts/clean_release_tui.py index 1cd820bc..3590db4d 100644 --- a/backend/src/scripts/clean_release_tui.py +++ b/backend/src/scripts/clean_release_tui.py @@ -9,9 +9,9 @@ import curses import json import os import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from types import SimpleNamespace -from typing import List, Optional, Any, Dict +from typing import Any # Standardize sys.path for direct execution from project root or scripts dir. SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -28,11 +28,11 @@ from src.models.clean_release import ( CleanProfilePolicy, ComplianceViolation, ProfileType, + RegistryStatus, ReleaseCandidate, + ReleaseCandidateStatus, ResourceSourceEntry, ResourceSourceRegistry, - RegistryStatus, - ReleaseCandidateStatus, ) from src.services.clean_release.approval_service import approve_candidate from src.services.clean_release.artifact_catalog_loader import load_bootstrap_artifacts @@ -124,7 +124,7 @@ class TuiFacadeAdapter: created_by=actor, ) - def get_overview(self, *, candidate_id: str) -> Dict[str, Any]: + def get_overview(self, *, candidate_id: str) -> dict[str, Any]: candidate = self.repository.get_candidate(candidate_id) manifests = self.repository.get_manifests_by_candidate(candidate_id) latest_manifest = ( @@ -206,11 +206,11 @@ class CleanReleaseTUI: self.facade = TuiFacadeAdapter(self.repo) self.candidate_id = self._resolve_candidate_id() self.status: Any = "READY" - self.checks_progress: List[Dict[str, Any]] = [] - self.violations_list: List[ComplianceViolation] = [] - self.report_id: Optional[str] = None - self.last_error: Optional[str] = None - self.overview: Dict[str, Any] = {} + self.checks_progress: list[dict[str, Any]] = [] + self.violations_list: list[ComplianceViolation] = [] + self.report_id: str | None = None + self.last_error: str | None = None + self.overview: dict[str, Any] = {} self.refresh_overview() curses.start_color() @@ -230,7 +230,7 @@ class CleanReleaseTUI: return repo def _bootstrap_demo_repository(self, repository: CleanReleaseRepository) -> None: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) policy = CleanProfilePolicy( policy_id="POL-ENT-CLEAN", policy_version="1", @@ -240,7 +240,7 @@ class CleanReleaseTUI: prohibited_artifact_categories=["test-data"], effective_from=now, ) - setattr(policy, "immutable", True) + policy.immutable = True repository.save_policy(policy) registry = ResourceSourceRegistry( @@ -257,10 +257,10 @@ class CleanReleaseTUI: updated_at=now, updated_by="system", ) - setattr(registry, "immutable", True) - setattr(registry, "allowed_hosts", ["internal-repo.company.com"]) - setattr(registry, "allowed_schemes", ["https"]) - setattr(registry, "allowed_source_types", ["artifactory"]) + registry.immutable = True + registry.allowed_hosts = ["internal-repo.company.com"] + registry.allowed_schemes = ["https"] + registry.allowed_source_types = ["artifactory"] repository.save_registry(registry) candidate = ReleaseCandidate( id="2026.03.03-rc1", @@ -307,10 +307,10 @@ class CleanReleaseTUI: if not bootstrap_path: return - with open(bootstrap_path, "r", encoding="utf-8") as bootstrap_file: + with open(bootstrap_path, encoding="utf-8") as bootstrap_file: payload = json.load(bootstrap_file) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) candidate = ReleaseCandidate( id=payload.get("candidate_id", "candidate-1"), version=payload.get("version", "1.0.0"), @@ -523,7 +523,7 @@ class CleanReleaseTUI: result = self.facade.run_compliance( candidate_id=self.candidate_id, actor="operator" ) - except Exception as exc: # noqa: BLE001 + except Exception as exc: self.status = CheckFinalStatus.FAILED self.last_error = str(exc) self.refresh_screen() @@ -563,7 +563,7 @@ class CleanReleaseTUI: self.violations_list = [] self.checks_progress = [] self.last_error = f"Manifest built: {manifest.id}" - except Exception as exc: # noqa: BLE001 + except Exception as exc: self.last_error = str(exc) self.refresh_overview() self.refresh_screen() @@ -585,7 +585,7 @@ class CleanReleaseTUI: try: self.facade.approve_latest(candidate_id=self.candidate_id, actor="operator") self.last_error = None - except Exception as exc: # noqa: BLE001 + except Exception as exc: self.last_error = str(exc) self.refresh_overview() self.refresh_screen() @@ -598,7 +598,7 @@ class CleanReleaseTUI: try: self.facade.publish_latest(candidate_id=self.candidate_id, actor="operator") self.last_error = None - except Exception as exc: # noqa: BLE001 + except Exception as exc: self.last_error = str(exc) self.refresh_overview() self.refresh_screen() @@ -611,7 +611,7 @@ class CleanReleaseTUI: try: self.facade.publish_latest(candidate_id=self.candidate_id, actor="operator") self.last_error = None - except Exception as exc: # noqa: BLE001 + except Exception as exc: self.last_error = str(exc) self.refresh_overview() self.refresh_screen() diff --git a/backend/src/scripts/create_admin.py b/backend/src/scripts/create_admin.py index 5420fb06..1847e116 100644 --- a/backend/src/scripts/create_admin.py +++ b/backend/src/scripts/create_admin.py @@ -8,18 +8,18 @@ # # @INVARIANT: Admin user must have the "Admin" role. -import sys import argparse +import sys from pathlib import Path # Add src to path sys.path.append(str(Path(__file__).parent.parent.parent)) -from src.core.database import AuthSessionLocal, init_db from src.core.auth.security import get_password_hash -from src.models.auth import User, Role -from src.core.logger import logger, belief_scope from src.core.cot_logger import seed_trace_id +from src.core.database import AuthSessionLocal, init_db +from src.core.logger import belief_scope, logger +from src.models.auth import Role, User # #region create_admin [TYPE Function] diff --git a/backend/src/scripts/init_auth_db.py b/backend/src/scripts/init_auth_db.py index fb45685d..60f10c06 100644 --- a/backend/src/scripts/init_auth_db.py +++ b/backend/src/scripts/init_auth_db.py @@ -14,10 +14,10 @@ from pathlib import Path # Add src to path sys.path.append(str(Path(__file__).parent.parent.parent)) +from src.core.cot_logger import seed_trace_id from src.core.database import init_db from src.core.encryption_key import ensure_encryption_key -from src.core.logger import logger, belief_scope -from src.core.cot_logger import seed_trace_id +from src.core.logger import belief_scope, logger from src.scripts.seed_permissions import seed_permissions diff --git a/backend/src/scripts/migrate_sqlite_to_postgres.py b/backend/src/scripts/migrate_sqlite_to_postgres.py index 27eac51f..d0cce640 100644 --- a/backend/src/scripts/migrate_sqlite_to_postgres.py +++ b/backend/src/scripts/migrate_sqlite_to_postgres.py @@ -14,15 +14,15 @@ import argparse import json import os import sqlite3 +from collections.abc import Iterable from pathlib import Path -from typing import Any, Dict, Iterable, Optional +from typing import Any from sqlalchemy import create_engine, text from sqlalchemy.exc import SQLAlchemyError -from src.core.logger import belief_scope, logger from src.core.cot_logger import seed_trace_id - +from src.core.logger import belief_scope, logger # #region Constants [TYPE Section] DEFAULT_TARGET_URL = os.getenv( @@ -62,7 +62,7 @@ def _json_load_if_needed(value: Any) -> Any: # #region _find_legacy_config_path [TYPE Function] # @BRIEF Resolves the existing legacy config.json path from candidates. -def _find_legacy_config_path(explicit_path: Optional[str]) -> Optional[Path]: +def _find_legacy_config_path(explicit_path: str | None) -> Path | None: with belief_scope("_find_legacy_config_path"): if explicit_path: p = Path(explicit_path) @@ -162,7 +162,7 @@ def _ensure_target_schema(engine) -> None: # #region _migrate_config [TYPE Function] # @BRIEF Migrates legacy config.json into app_configurations(global). -def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int: +def _migrate_config(engine, legacy_config_path: Path | None) -> int: with belief_scope("_migrate_config"): if legacy_config_path is None: logger.reason( @@ -196,7 +196,7 @@ def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int: # #region _migrate_tasks_and_logs [TYPE Function] # @BRIEF Migrates task_records and task_logs from SQLite into PostgreSQL. -def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str, int]: +def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> dict[str, int]: with belief_scope("_migrate_tasks_and_logs"): stats = { "task_records_total": 0, @@ -326,8 +326,8 @@ def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str # #region run_migration [TYPE Function] # @BRIEF Orchestrates migration from SQLite/file to PostgreSQL. def run_migration( - sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path] -) -> Dict[str, int]: + sqlite_path: Path, target_url: str, legacy_config_path: Path | None +) -> dict[str, int]: seed_trace_id() with belief_scope("run_migration"): logger.reason( diff --git a/backend/src/scripts/seed_permissions.py b/backend/src/scripts/seed_permissions.py index 398bd35f..0fd9d2c2 100644 --- a/backend/src/scripts/seed_permissions.py +++ b/backend/src/scripts/seed_permissions.py @@ -15,11 +15,11 @@ from pathlib import Path # Add src to path sys.path.append(str(Path(__file__).parent.parent.parent)) -from src.core.database import AuthSessionLocal -from src.models.auth import Permission, Role from src.core.auth.repository import AuthRepository -from src.core.logger import logger, belief_scope from src.core.cot_logger import seed_trace_id +from src.core.database import AuthSessionLocal +from src.core.logger import belief_scope, logger +from src.models.auth import Permission, Role # #region INITIAL_PERMISSIONS [C:3] [TYPE Constant] # @BRIEF Canonical bootstrap permission tuples seeded into auth storage. diff --git a/backend/src/scripts/seed_superset_load_test.py b/backend/src/scripts/seed_superset_load_test.py index c077ce57..b9607017 100644 --- a/backend/src/scripts/seed_superset_load_test.py +++ b/backend/src/scripts/seed_superset_load_test.py @@ -12,14 +12,13 @@ import random import sys import uuid from pathlib import Path -from typing import Dict, List, Optional sys.path.append(str(Path(__file__).parent.parent.parent)) from src.core.config_manager import ConfigManager from src.core.config_models import Environment -from src.core.logger import belief_scope, logger from src.core.cot_logger import seed_trace_id +from src.core.logger import belief_scope, logger from src.core.superset_client import SupersetClient @@ -71,7 +70,7 @@ def _parse_args() -> argparse.Namespace: # @BRIEF Normalizes Superset API payloads that may be wrapped in `result`. # @PRE: payload is a JSON-decoded API response. # @POST: Returns the unwrapped object when present. -def _extract_result_payload(payload: Dict) -> Dict: +def _extract_result_payload(payload: dict) -> dict: result = payload.get("result") if isinstance(result, dict): return result @@ -85,7 +84,7 @@ def _extract_result_payload(payload: Dict) -> Dict: # @BRIEF Extracts object ID from create/update API response. # @PRE: payload is a JSON-decoded API response. # @POST: Returns integer object ID or None if missing. -def _extract_created_id(payload: Dict) -> Optional[int]: +def _extract_created_id(payload: dict) -> int | None: direct_id = payload.get("id") if isinstance(direct_id, int): return direct_id @@ -142,10 +141,10 @@ def _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) # @BRIEF Resolves requested environment IDs from configuration. # @PRE: env_ids is non-empty. # @POST: Returns mapping env_id -> configured environment object. -def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]: +def _resolve_target_envs(env_ids: list[str]) -> dict[str, Environment]: config_manager = ConfigManager() configured = {env.id: env for env in config_manager.get_environments()} - resolved: Dict[str, Environment] = {} + resolved: dict[str, Environment] = {} if not configured: for config_path in [Path("config.json"), Path("backend/config.json")]: @@ -181,7 +180,7 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]: # @POST: Returns non-empty list of chart payload templates. def _build_chart_template_pool( client: SupersetClient, pool_size: int, rng: random.Random -) -> List[Dict]: +) -> list[dict]: list_query = { "page": 0, "page_size": 1000, @@ -209,7 +208,7 @@ def _build_chart_template_pool( if len(candidates) <= pool_size else rng.sample(candidates, pool_size) ) - templates: List[Dict] = [] + templates: list[dict] = [] for row in selected: chart_id = int(row["id"]) @@ -255,15 +254,15 @@ def _build_chart_template_pool( # @PRE: Target environments must be reachable and authenticated. # @POST: Returns execution statistics dictionary. # @SIDE_EFFECT: Creates objects in Superset environments. -def seed_superset_load_data(args: argparse.Namespace) -> Dict: +def seed_superset_load_data(args: argparse.Namespace) -> dict: seed_trace_id() rng = random.Random(args.seed) env_map = _resolve_target_envs(args.envs) - clients: Dict[str, SupersetClient] = {} - templates_by_env: Dict[str, List[Dict]] = {} - created_dashboards: Dict[str, List[int]] = {env_id: [] for env_id in env_map} - created_charts: Dict[str, List[int]] = {env_id: [] for env_id in env_map} + clients: dict[str, SupersetClient] = {} + templates_by_env: dict[str, list[dict]] = {} + created_dashboards: dict[str, list[int]] = {env_id: [] for env_id in env_map} + created_charts: dict[str, list[int]] = {env_id: [] for env_id in env_map} used_chart_names: set[str] = set() used_dashboard_names: set[str] = set() diff --git a/backend/src/scripts/test_dataset_dashboard_relations.py b/backend/src/scripts/test_dataset_dashboard_relations.py index 5cdbb1c2..35cf2a87 100644 --- a/backend/src/scripts/test_dataset_dashboard_relations.py +++ b/backend/src/scripts/test_dataset_dashboard_relations.py @@ -15,46 +15,46 @@ from pathlib import Path # Add src to path (parent of scripts directory) sys.path.append(str(Path(__file__).parent.parent.parent)) -from src.core.superset_client import SupersetClient from src.core.config_manager import ConfigManager -from src.core.logger import logger from src.core.cot_logger import seed_trace_id +from src.core.logger import logger +from src.core.superset_client import SupersetClient def test_dashboard_dataset_relations(): """Test fetching dataset-to-dashboard relationships.""" seed_trace_id() - + # Load environment from existing config config_manager = ConfigManager() environments = config_manager.get_environments() - + if not environments: logger.error("No environments configured!") return - + # Use first available environment env = environments[0] logger.info(f"Using environment: {env.name} ({env.url})") - + client = SupersetClient(env) - + try: # Authenticate logger.info("Authenticating to Superset...") client.authenticate() logger.info("Authentication successful!") - + # Test dashboard ID 13 dashboard_id = 13 logger.info(f"\n=== Fetching Dashboard {dashboard_id} ===") dashboard = client.network.request(method="GET", endpoint=f"/dashboard/{dashboard_id}") - + print("\nDashboard structure:") print(f" ID: {dashboard.get('id')}") print(f" Title: {dashboard.get('dashboard_title')}") print(f" Published: {dashboard.get('published')}") - + # Check for slices/charts if 'slices' in dashboard: logger.info(f"\n Found {len(dashboard['slices'])} slices/charts in dashboard") @@ -72,21 +72,21 @@ def test_dashboard_dataset_relations(): else: logger.warning(" No 'slices' field found in dashboard response") logger.info(f" Available fields: {list(dashboard.keys())}") - + # Test dataset ID 26 dataset_id = 26 logger.info(f"\n=== Fetching Dataset {dataset_id} ===") dataset = client.get_dataset(dataset_id) - + print("\nDataset structure:") print(f" ID: {dataset.get('id')}") print(f" Table Name: {dataset.get('table_name')}") print(f" Schema: {dataset.get('schema')}") print(f" Database: {dataset.get('database', {}).get('database_name', 'Unknown')}") - + # Check for dashboards that use this dataset logger.info(f"\n=== Finding Dashboards using Dataset {dataset_id} ===") - + # Method: Use Superset's related_objects API try: logger.info(f" Using /api/v1/dataset/{dataset_id}/related_objects endpoint...") @@ -94,15 +94,15 @@ def test_dashboard_dataset_relations(): method="GET", endpoint=f"/dataset/{dataset_id}/related_objects" ) - + logger.info(f" Related objects response type: {type(related_objects)}") logger.info(f" Related objects keys: {list(related_objects.keys()) if isinstance(related_objects, dict) else 'N/A'}") - + # Check for dashboards in related objects if 'dashboards' in related_objects: dashboards = related_objects['dashboards'] logger.info(f" Found {len(dashboards)} dashboards using this dataset:") - + for dash in dashboards: if isinstance(dash, dict): logger.info(f" - Dashboard ID {dash.get('id')}: {dash.get('dashboard_title', dash.get('title', 'Unknown'))}") @@ -114,27 +114,27 @@ def test_dashboard_dataset_relations(): if 'dashboards' in result: dashboards = result['dashboards'] logger.info(f" Found {len(dashboards)} dashboards using this dataset:") - + for dash in dashboards: logger.info(f" - Dashboard ID {dash.get('id')}: {dash.get('dashboard_title', dash.get('title', 'Unknown'))}") else: logger.warning(f" No 'dashboards' key in result. Keys: {list(result.keys())}") else: logger.warning(f" No 'dashboards' key in response. Available keys: {list(related_objects.keys())}") - logger.info(f" Full related_objects response:") + logger.info(" Full related_objects response:") print(json.dumps(related_objects, indent=2, default=str)[:1000]) - + except Exception as e: logger.error(f" Error fetching related objects: {e}") import traceback traceback.print_exc() - + # Method 2: Try to use the position_json from dashboard - logger.info(f"\n=== Analyzing Dashboard Position JSON ===") + logger.info("\n=== Analyzing Dashboard Position JSON ===") if 'position_json' in dashboard: position_data = json.loads(dashboard['position_json']) logger.info(f" Position data type: {type(position_data)}") - + # Look for datasource references datasource_ids = set() if isinstance(position_data, dict): @@ -148,10 +148,10 @@ def test_dashboard_dataset_relations(): if isinstance(item, dict): if 'datasource_id' in item: datasource_ids.add(item['datasource_id']) - + if datasource_ids: logger.info(f" Found datasource IDs: {datasource_ids}") - + # Save full response for analysis output_file = Path(__file__).parent / "dataset_dashboard_analysis.json" with open(output_file, 'w') as f: @@ -160,7 +160,7 @@ def test_dashboard_dataset_relations(): 'dataset': dataset }, f, indent=2, default=str) logger.info(f"\nFull response saved to: {output_file}") - + except Exception as e: logger.error(f"Error: {e}", exc_info=True) raise diff --git a/backend/src/services/__tests__/test_encryption_manager.py b/backend/src/services/__tests__/test_encryption_manager.py index 84d36c66..d45ce44c 100644 --- a/backend/src/services/__tests__/test_encryption_manager.py +++ b/backend/src/services/__tests__/test_encryption_manager.py @@ -7,11 +7,13 @@ import sys from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest from unittest.mock import patch -from cryptography.fernet import Fernet, InvalidToken + +import pytest +from cryptography.fernet import Fernet # region TestEncryptionManager [TYPE Class] @@ -105,9 +107,8 @@ class TestEncryptionManager: def test_missing_key_fails_fast(self): from src.services.llm_provider import EncryptionManager - with patch.dict("os.environ", {}, clear=True): - with pytest.raises(RuntimeError, match="ENCRYPTION_KEY must be set"): - EncryptionManager() + with patch.dict("os.environ", {}, clear=True), pytest.raises(RuntimeError, match="ENCRYPTION_KEY must be set"): + EncryptionManager() # endregion test_missing_key_fails_fast # region test_custom_key_roundtrip [TYPE Function] diff --git a/backend/src/services/__tests__/test_health_service.py b/backend/src/services/__tests__/test_health_service.py index ff5cf371..ea737c37 100644 --- a/backend/src/services/__tests__/test_health_service.py +++ b/backend/src/services/__tests__/test_health_service.py @@ -1,8 +1,10 @@ -import pytest from datetime import datetime, timedelta from unittest.mock import MagicMock, patch -from src.services.health_service import HealthService + +import pytest + from src.models.llm import ValidationRecord +from src.services.health_service import HealthService # region test_health_service [TYPE Module] # @PURPOSE: Unit tests for HealthService aggregation logic. diff --git a/backend/src/services/__tests__/test_llm_plugin_persistence.py b/backend/src/services/__tests__/test_llm_plugin_persistence.py index 525b787b..1ae7d991 100644 --- a/backend/src/services/__tests__/test_llm_plugin_persistence.py +++ b/backend/src/services/__tests__/test_llm_plugin_persistence.py @@ -3,6 +3,7 @@ # @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context. import types + import pytest from src.plugins.llm_analysis import plugin as plugin_module diff --git a/backend/src/services/__tests__/test_llm_prompt_templates.py b/backend/src/services/__tests__/test_llm_prompt_templates.py index b71ef356..2afbe533 100644 --- a/backend/src/services/__tests__/test_llm_prompt_templates.py +++ b/backend/src/services/__tests__/test_llm_prompt_templates.py @@ -7,12 +7,12 @@ from src.services.llm_prompt_templates import ( DEFAULT_LLM_ASSISTANT_SETTINGS, - DEFAULT_LLM_PROVIDER_BINDINGS, DEFAULT_LLM_PROMPTS, + DEFAULT_LLM_PROVIDER_BINDINGS, is_multimodal_model, normalize_llm_settings, - resolve_bound_provider_id, render_prompt, + resolve_bound_provider_id, ) diff --git a/backend/src/services/__tests__/test_llm_provider.py b/backend/src/services/__tests__/test_llm_provider.py index 36fe3119..9cca56bf 100644 --- a/backend/src/services/__tests__/test_llm_provider.py +++ b/backend/src/services/__tests__/test_llm_provider.py @@ -4,14 +4,16 @@ # @PURPOSE: Contract testing for LLMProviderService and EncryptionManager # endregion test_llm_provider -import pytest import os from unittest.mock import MagicMock -from sqlalchemy.orm import Session + +import pytest from cryptography.fernet import Fernet -from src.services.llm_provider import EncryptionManager, LLMProviderService +from sqlalchemy.orm import Session + from src.models.llm import LLMProvider from src.plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType +from src.services.llm_provider import EncryptionManager, LLMProviderService # region _test_encryption_key_fixture [TYPE Global] # @PURPOSE: Ensure encryption-dependent provider tests run with a valid Fernet key. diff --git a/backend/src/services/__tests__/test_rbac_permission_catalog.py b/backend/src/services/__tests__/test_rbac_permission_catalog.py index c4f0a3e6..006abfd5 100644 --- a/backend/src/services/__tests__/test_rbac_permission_catalog.py +++ b/backend/src/services/__tests__/test_rbac_permission_catalog.py @@ -10,6 +10,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock import src.services.rbac_permission_catalog as catalog + # [/SECTION: IMPORTS] @@ -140,4 +141,4 @@ def test_sync_permission_catalog_is_noop_when_all_permissions_exist(): # endregion test_sync_permission_catalog_is_noop_when_all_permissions_exist -# endregion test_rbac_permission_catalog \ No newline at end of file +# endregion test_rbac_permission_catalog diff --git a/backend/src/services/__tests__/test_resource_service.py b/backend/src/services/__tests__/test_resource_service.py index a3ef60f5..1b3e8554 100644 --- a/backend/src/services/__tests__/test_resource_service.py +++ b/backend/src/services/__tests__/test_resource_service.py @@ -5,9 +5,10 @@ # @RELATION: VERIFIES ->[src.services.resource_service.ResourceService] # @INVARIANT: Resource summaries preserve task linkage and status projection behavior. +from datetime import UTC, datetime +from unittest.mock import MagicMock, patch + import pytest -from unittest.mock import MagicMock, patch, AsyncMock -from datetime import datetime, timezone # region test_get_dashboards_with_status [TYPE Function] @@ -316,7 +317,7 @@ async def test_get_dashboards_with_status_handles_mixed_naive_and_aware_task_dat task_aware.plugin_id = "llm_dashboard_validation" task_aware.status = "SUCCESS" task_aware.params = {"dashboard_id": "1", "environment_id": "prod"} - task_aware.started_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + task_aware.started_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) env = MagicMock() env.id = "prod" @@ -449,7 +450,7 @@ def test_get_last_task_for_resource_handles_mixed_naive_and_aware_created_at(): task_aware.id = "task-new" task_aware.status = "RUNNING" task_aware.params = {"resource_id": "dashboard-1"} - task_aware.created_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + task_aware.created_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) result = service._get_last_task_for_resource( "dashboard-1", [task_naive, task_aware] diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index 5ad88348..1673b946 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -12,16 +12,17 @@ # @SIDE_EFFECT: Writes last login timestamps and JIT-provisions external users. # @DATA_CONTRACT: [Credentials | ADFSClaims] -> [UserEntity | SessionToken] -from typing import Dict, Any, Optional, List from datetime import datetime +from typing import Any + from sqlalchemy.orm import Session -from ..core.auth.repository import AuthRepository -from ..core.auth.security import verify_password from ..core.auth.jwt import create_access_token from ..core.auth.logger import log_security_event -from ..models.auth import User, Role +from ..core.auth.repository import AuthRepository +from ..core.auth.security import verify_password from ..core.logger import belief_scope +from ..models.auth import User # #region AuthService [C:3] [TYPE Class] @@ -55,7 +56,7 @@ class AuthService: # @PARAM: username (str) - The username. # @PARAM: password (str) - The plain password. # @RETURN: Optional[User] - The authenticated user or None. - def authenticate_user(self, username: str, password: str) -> Optional[User]: + def authenticate_user(self, username: str, password: str) -> User | None: with belief_scope("auth.authenticate_user"): user = self.repo.get_user_by_username(username) if not user or not user.is_active: @@ -84,7 +85,7 @@ class AuthService: # @RELATION: [DEPENDS_ON] ->[Role] # @PARAM: user (User) - The authenticated user. # @RETURN: Dict[str, str] - Session data. - def create_session(self, user: User) -> Dict[str, str]: + def create_session(self, user: User) -> dict[str, str]: with belief_scope("auth.create_session"): roles = [role.name for role in user.roles] access_token = create_access_token( @@ -105,7 +106,7 @@ class AuthService: # @RELATION: [DEPENDS_ON] ->[Role] # @PARAM: user_info (Dict[str, Any]) - Claims from ADFS token. # @RETURN: User - The provisioned user. - def provision_adfs_user(self, user_info: Dict[str, Any]) -> User: + def provision_adfs_user(self, user_info: dict[str, Any]) -> User: with belief_scope("auth.provision_adfs_user"): username = user_info.get("upn") or user_info.get("email") email = user_info.get("email") diff --git a/backend/src/services/clean_release/__tests__/test_audit_service.py b/backend/src/services/clean_release/__tests__/test_audit_service.py index 2f6f5333..719aa036 100644 --- a/backend/src/services/clean_release/__tests__/test_audit_service.py +++ b/backend/src/services/clean_release/__tests__/test_audit_service.py @@ -5,9 +5,10 @@ # @LAYER: Infra from unittest.mock import patch + from src.services.clean_release.audit_service import ( - audit_preparation, audit_check_run, + audit_preparation, audit_report, ) diff --git a/backend/src/services/clean_release/__tests__/test_policy_engine.py b/backend/src/services/clean_release/__tests__/test_policy_engine.py index dddbebf8..4d3522de 100644 --- a/backend/src/services/clean_release/__tests__/test_policy_engine.py +++ b/backend/src/services/clean_release/__tests__/test_policy_engine.py @@ -3,14 +3,16 @@ # @PURPOSE: Contract testing for CleanPolicyEngine # endregion TestPolicyEngine -import pytest from datetime import datetime + +import pytest + from src.models.clean_release import ( CleanProfilePolicy, - ResourceSourceRegistry, - ResourceSourceEntry, ProfileType, RegistryStatus, + ResourceSourceEntry, + ResourceSourceRegistry, ) from src.services.clean_release.policy_engine import CleanPolicyEngine diff --git a/backend/src/services/clean_release/__tests__/test_preparation_service.py b/backend/src/services/clean_release/__tests__/test_preparation_service.py index 3aaf5110..fed00ea9 100644 --- a/backend/src/services/clean_release/__tests__/test_preparation_service.py +++ b/backend/src/services/clean_release/__tests__/test_preparation_service.py @@ -5,18 +5,18 @@ # @RELATION: [DEPENDS_ON] ->[PreparationService] # @INVARIANT: Candidate preparation always persists manifest and candidate status deterministically. -import pytest +from datetime import UTC, datetime from unittest.mock import MagicMock, patch -from datetime import datetime, timezone + +import pytest from src.models.clean_release import ( CleanProfilePolicy, - ResourceSourceRegistry, - ResourceSourceEntry, + ProfileType, ReleaseCandidate, ReleaseCandidateStatus, - ProfileType, - DistributionManifest, + ResourceSourceEntry, + ResourceSourceRegistry, ) from src.services.clean_release.preparation_service import prepare_candidate @@ -33,7 +33,7 @@ def _mock_policy() -> CleanProfilePolicy: required_system_categories=["system"], external_source_forbidden=True, internal_source_registry_ref="reg-1", - effective_from=datetime.now(timezone.utc), + effective_from=datetime.now(UTC), profile=ProfileType.ENTERPRISE_CLEAN, ) @@ -57,7 +57,7 @@ def _mock_registry() -> ResourceSourceRegistry: enabled=True, ) ], - updated_at=datetime.now(timezone.utc), + updated_at=datetime.now(UTC), updated_by="tester", ) @@ -73,7 +73,7 @@ def _mock_candidate(candidate_id: str) -> ReleaseCandidate: candidate_id=candidate_id, version="1.0.0", profile=ProfileType.ENTERPRISE_CLEAN, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=ReleaseCandidateStatus.DRAFT, created_by="tester", source_snapshot_ref="v1.0.0-snapshot", diff --git a/backend/src/services/clean_release/__tests__/test_report_builder.py b/backend/src/services/clean_release/__tests__/test_report_builder.py index 03e791ad..ecd70f9e 100644 --- a/backend/src/services/clean_release/__tests__/test_report_builder.py +++ b/backend/src/services/clean_release/__tests__/test_report_builder.py @@ -5,7 +5,7 @@ # @LAYER: Domain # @INVARIANT: blocked run requires at least one blocking violation. -from datetime import datetime, timezone +from datetime import UTC, datetime import pytest @@ -29,8 +29,8 @@ def _terminal_run(status: CheckFinalStatus) -> ComplianceCheckRun: check_run_id="check-1", candidate_id="2026.03.03-rc1", policy_id="policy-enterprise-clean-v1", - started_at=datetime.now(timezone.utc), - finished_at=datetime.now(timezone.utc), + started_at=datetime.now(UTC), + finished_at=datetime.now(UTC), final_status=status, triggered_by="tester", execution_mode=ExecutionMode.TUI, @@ -53,7 +53,7 @@ def _blocking_violation() -> ComplianceViolation: location="pypi.org", remediation="replace", blocked_release=True, - detected_at=datetime.now(timezone.utc), + detected_at=datetime.now(UTC), ) diff --git a/backend/src/services/clean_release/__tests__/test_source_isolation.py b/backend/src/services/clean_release/__tests__/test_source_isolation.py index 59fe1a8c..8e87640f 100644 --- a/backend/src/services/clean_release/__tests__/test_source_isolation.py +++ b/backend/src/services/clean_release/__tests__/test_source_isolation.py @@ -5,7 +5,7 @@ # @LAYER: Domain # @INVARIANT: External endpoints always produce blocking violations. -from datetime import datetime, timezone +from datetime import UTC, datetime from src.models.clean_release import ResourceSourceEntry, ResourceSourceRegistry from src.services.clean_release.source_isolation import validate_internal_sources @@ -33,7 +33,7 @@ def _registry() -> ResourceSourceRegistry: enabled=True, ), ], - updated_at=datetime.now(timezone.utc), + updated_at=datetime.now(UTC), updated_by="tester", status="active", ) diff --git a/backend/src/services/clean_release/__tests__/test_stages.py b/backend/src/services/clean_release/__tests__/test_stages.py index ac409287..2a0d742b 100644 --- a/backend/src/services/clean_release/__tests__/test_stages.py +++ b/backend/src/services/clean_release/__tests__/test_stages.py @@ -6,11 +6,10 @@ from src.models.clean_release import ( CheckFinalStatus, - CheckStageName, CheckStageResult, CheckStageStatus, ) -from src.services.clean_release.stages import derive_final_status, MANDATORY_STAGE_ORDER +from src.services.clean_release.stages import MANDATORY_STAGE_ORDER, derive_final_status # region test_derive_final_status_compliant [TYPE Function] diff --git a/backend/src/services/clean_release/approval_service.py b/backend/src/services/clean_release/approval_service.py index 3c8c4a8b..ea3537a0 100644 --- a/backend/src/services/clean_release/approval_service.py +++ b/backend/src/services/clean_release/approval_service.py @@ -8,8 +8,7 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import List +from datetime import UTC, datetime from uuid import uuid4 from ...core.logger import belief_scope, logger @@ -26,11 +25,11 @@ from .repository import CleanReleaseRepository # @POST: Returns mutable decision list attached to repository. def _get_or_init_decisions_store( repository: CleanReleaseRepository, -) -> List[ApprovalDecision]: +) -> list[ApprovalDecision]: decisions = getattr(repository, "approval_decisions", None) if decisions is None: decisions = [] - setattr(repository, "approval_decisions", decisions) + repository.approval_decisions = decisions return decisions @@ -50,7 +49,7 @@ def _latest_decision_for_candidate( return None return sorted( scoped, - key=lambda item: item.decided_at or datetime.min.replace(tzinfo=timezone.utc), + key=lambda item: item.decided_at or datetime.min.replace(tzinfo=UTC), reverse=True, )[0] @@ -133,7 +132,7 @@ def approve_candidate( repository.save_candidate(candidate) except ApprovalGateError: raise - except Exception as exc: # noqa: BLE001 + except Exception as exc: logger.explore( f"Candidate transition to APPROVED failed candidate_id={candidate_id}: {exc}" ) @@ -145,7 +144,7 @@ def approve_candidate( report_id=report_id, decision=ApprovalDecisionType.APPROVED.value, decided_by=decided_by, - decided_at=datetime.now(timezone.utc), + decided_at=datetime.now(UTC), comment=comment, ) _get_or_init_decisions_store(repository).append(decision) @@ -193,7 +192,7 @@ def reject_candidate( report_id=report_id, decision=ApprovalDecisionType.REJECTED.value, decided_by=decided_by, - decided_at=datetime.now(timezone.utc), + decided_at=datetime.now(UTC), comment=comment, ) _get_or_init_decisions_store(repository).append(decision) diff --git a/backend/src/services/clean_release/artifact_catalog_loader.py b/backend/src/services/clean_release/artifact_catalog_loader.py index c6c07396..2aa41a15 100644 --- a/backend/src/services/clean_release/artifact_catalog_loader.py +++ b/backend/src/services/clean_release/artifact_catalog_loader.py @@ -8,7 +8,6 @@ from __future__ import annotations import json from pathlib import Path -from typing import Any, Dict, List from ...models.clean_release import CandidateArtifact @@ -17,7 +16,7 @@ from ...models.clean_release import CandidateArtifact # @BRIEF Parse artifact catalog JSON into CandidateArtifact models for TUI/bootstrap flows. # @PRE: path points to readable JSON file; payload is list[artifact] or {"artifacts": list[artifact]}. # @POST: Returns non-mutated CandidateArtifact models with required fields populated. -def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArtifact]: +def load_bootstrap_artifacts(path: str, candidate_id: str) -> list[CandidateArtifact]: if not path or not path.strip(): return [] if not candidate_id or not candidate_id.strip(): @@ -31,7 +30,7 @@ def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArti "artifact catalog must be a list or an object with 'artifacts' list" ) - artifacts: List[CandidateArtifact] = [] + artifacts: list[CandidateArtifact] = [] for index, raw_artifact in enumerate(raw_artifacts, start=1): if not isinstance(raw_artifact, dict): raise ValueError(f"artifact #{index} must be an object") diff --git a/backend/src/services/clean_release/audit_service.py b/backend/src/services/clean_release/audit_service.py index a42d69e8..3aa18e4a 100644 --- a/backend/src/services/clean_release/audit_service.py +++ b/backend/src/services/clean_release/audit_service.py @@ -6,14 +6,14 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import Any, Dict, Optional +from datetime import UTC, datetime +from typing import Any from uuid import uuid4 from ...core.logger import logger -def _append_event(repository, payload: Dict[str, Any]) -> None: +def _append_event(repository, payload: dict[str, Any]) -> None: if repository is not None and hasattr(repository, "append_audit_event"): repository.append_audit_event(payload) @@ -32,7 +32,7 @@ def audit_preparation( "candidate_id": candidate_id, "actor": actor, "status": status, - "timestamp": datetime.now(timezone.utc).isoformat(), + "timestamp": datetime.now(UTC).isoformat(), }, ) @@ -42,7 +42,7 @@ def audit_check_run( final_status: str, repository=None, *, - candidate_id: Optional[str] = None, + candidate_id: str | None = None, actor: str = "system", ) -> None: logger.reflect( @@ -57,7 +57,7 @@ def audit_check_run( "candidate_id": candidate_id, "actor": actor, "status": final_status, - "timestamp": datetime.now(timezone.utc).isoformat(), + "timestamp": datetime.now(UTC).isoformat(), }, ) @@ -68,7 +68,7 @@ def audit_violation( code: str, repository=None, *, - candidate_id: Optional[str] = None, + candidate_id: str | None = None, actor: str = "system", ) -> None: logger.explore( @@ -84,7 +84,7 @@ def audit_violation( "actor": actor, "stage_name": stage_name, "code": code, - "timestamp": datetime.now(timezone.utc).isoformat(), + "timestamp": datetime.now(UTC).isoformat(), }, ) @@ -94,7 +94,7 @@ def audit_report( candidate_id: str, repository=None, *, - run_id: Optional[str] = None, + run_id: str | None = None, actor: str = "system", ) -> None: logger.explore( @@ -109,7 +109,7 @@ def audit_report( "run_id": run_id, "candidate_id": candidate_id, "actor": actor, - "timestamp": datetime.now(timezone.utc).isoformat(), + "timestamp": datetime.now(UTC).isoformat(), }, ) diff --git a/backend/src/services/clean_release/candidate_service.py b/backend/src/services/clean_release/candidate_service.py index 240e2f87..5bc7ffc6 100644 --- a/backend/src/services/clean_release/candidate_service.py +++ b/backend/src/services/clean_release/candidate_service.py @@ -9,8 +9,9 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import Any, Dict, Iterable, List +from collections.abc import Iterable +from datetime import UTC, datetime +from typing import Any from ...models.clean_release import CandidateArtifact, ReleaseCandidate from .enums import CandidateStatus @@ -21,7 +22,7 @@ from .repository import CleanReleaseRepository # @BRIEF Validate raw artifact payload list for required fields and shape. # @PRE: artifacts payload is provided by caller. # @POST: Returns normalized artifact list or raises ValueError. -def _validate_artifacts(artifacts: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]: +def _validate_artifacts(artifacts: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: normalized = list(artifacts) if not normalized: raise ValueError("artifacts must not be empty") @@ -55,7 +56,7 @@ def register_candidate( version: str, source_snapshot_ref: str, created_by: str, - artifacts: Iterable[Dict[str, Any]], + artifacts: Iterable[dict[str, Any]], ) -> ReleaseCandidate: if not candidate_id or not candidate_id.strip(): raise ValueError("candidate_id must be non-empty") @@ -77,7 +78,7 @@ def register_candidate( version=version, source_snapshot_ref=source_snapshot_ref, created_by=created_by, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.DRAFT.value, ) repository.save_candidate(candidate) @@ -102,4 +103,4 @@ def register_candidate( return candidate # #endregion register_candidate -# #endregion backend.src.services.clean_release.candidate_service \ No newline at end of file +# #endregion backend.src.services.clean_release.candidate_service diff --git a/backend/src/services/clean_release/compliance_execution_service.py b/backend/src/services/clean_release/compliance_execution_service.py index f840731c..9b9dda99 100644 --- a/backend/src/services/clean_release/compliance_execution_service.py +++ b/backend/src/services/clean_release/compliance_execution_service.py @@ -13,9 +13,10 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Any, Iterable, List, Optional, cast +from datetime import UTC, datetime +from typing import Any, cast from uuid import uuid4 from ...core.logger import belief_scope, logger @@ -26,7 +27,6 @@ from ...models.clean_release import ( ComplianceViolation, DistributionManifest, ) -from .audit_service import audit_check_run, audit_report, audit_violation from .enums import ComplianceDecision, RunStatus from .exceptions import ComplianceRunError, PolicyResolutionError from .policy_resolution_service import resolve_trusted_policy_snapshots @@ -35,7 +35,6 @@ from .repository import CleanReleaseRepository from .stages import build_default_stages, derive_final_status from .stages.base import ComplianceStage, ComplianceStageContext, build_stage_run_record - belief_logger = cast(Any, logger) @@ -44,9 +43,9 @@ belief_logger = cast(Any, logger) @dataclass class ComplianceExecutionResult: run: ComplianceRun - report: Optional[ComplianceReport] - stage_runs: List[ComplianceStageRun] - violations: List[ComplianceViolation] + report: ComplianceReport | None + stage_runs: list[ComplianceStageRun] + violations: list[ComplianceViolation] # #endregion ComplianceExecutionResult @@ -66,7 +65,7 @@ class ComplianceExecutionService: *, repository: CleanReleaseRepository, config_manager, - stages: Optional[Iterable[ComplianceStage]] = None, + stages: Iterable[ComplianceStage] | None = None, ): self.repository = repository self.config_manager = config_manager @@ -78,17 +77,17 @@ class ComplianceExecutionService: # @PRE: candidate exists. # @POST: Returns manifest snapshot or raises ComplianceRunError. def _resolve_manifest( - self, candidate_id: str, manifest_id: Optional[str] + self, candidate_id: str, manifest_id: str | None ) -> DistributionManifest: with belief_scope("ComplianceExecutionService._resolve_manifest"): - manifest_id_value = cast(Optional[str], manifest_id) + manifest_id_value = cast(str | None, manifest_id) if manifest_id_value is not None and manifest_id_value != "": manifest = self.repository.get_manifest(manifest_id_value) if manifest is None: raise ComplianceRunError( f"manifest '{manifest_id_value}' not found" ) - if str(getattr(manifest, "candidate_id")) != candidate_id: + if str(manifest.candidate_id) != candidate_id: raise ComplianceRunError("manifest does not belong to candidate") return manifest @@ -112,7 +111,7 @@ class ComplianceExecutionService: # region _persist_violations [TYPE Function] # @PURPOSE: Persist stage violations via repository adapters. # @POST: Violations are appended to repository evidence store. - def _persist_violations(self, violations: List[ComplianceViolation]) -> None: + def _persist_violations(self, violations: list[ComplianceViolation]) -> None: for violation in violations: self.repository.save_violation(violation) @@ -127,7 +126,7 @@ class ComplianceExecutionService: *, candidate_id: str, requested_by: str, - manifest_id: Optional[str] = None, + manifest_id: str | None = None, ) -> ComplianceExecutionResult: with belief_scope("ComplianceExecutionService.execute_run"): belief_logger.reason( @@ -151,20 +150,20 @@ class ComplianceExecutionService: run = ComplianceRun( id=f"run-{uuid4()}", candidate_id=candidate_id, - manifest_id=str(getattr(manifest, "id")), - manifest_digest=str(getattr(manifest, "manifest_digest")), - policy_snapshot_id=str(getattr(policy_snapshot, "id")), - registry_snapshot_id=str(getattr(registry_snapshot, "id")), + manifest_id=str(manifest.id), + manifest_digest=str(manifest.manifest_digest), + policy_snapshot_id=str(policy_snapshot.id), + registry_snapshot_id=str(registry_snapshot.id), requested_by=requested_by, - requested_at=datetime.now(timezone.utc), - started_at=datetime.now(timezone.utc), + requested_at=datetime.now(UTC), + started_at=datetime.now(UTC), status=RunStatus.RUNNING.value, ) self.repository.save_check_run(run) - stage_runs: List[ComplianceStageRun] = [] - violations: List[ComplianceViolation] = [] - report: Optional[ComplianceReport] = None + stage_runs: list[ComplianceStageRun] = [] + violations: list[ComplianceViolation] = [] + report: ComplianceReport | None = None context = ComplianceStageContext( run=run, @@ -176,12 +175,12 @@ class ComplianceExecutionService: try: for stage in self.stages: - started = datetime.now(timezone.utc) + started = datetime.now(UTC) result = stage.execute(context) - finished = datetime.now(timezone.utc) + finished = datetime.now(UTC) stage_run = build_stage_run_record( - run_id=str(getattr(run, "id")), + run_id=str(run.id), stage_name=stage.stage_name, result=result, started_at=started, @@ -194,26 +193,26 @@ class ComplianceExecutionService: self._persist_violations(result.violations) violations.extend(result.violations) - setattr(run, "final_status", derive_final_status(stage_runs).value) - setattr(run, "status", RunStatus.SUCCEEDED.value) - setattr(run, "finished_at", datetime.now(timezone.utc)) + run.final_status = derive_final_status(stage_runs).value + run.status = RunStatus.SUCCEEDED.value + run.finished_at = datetime.now(UTC) self.repository.save_check_run(run) report = self.report_builder.build_report_payload(run, violations) report = self.report_builder.persist_report(report) - setattr(run, "report_id", str(getattr(report, "id"))) + run.report_id = str(report.id) self.repository.save_check_run(run) belief_logger.reflect( - f"Compliance run completed run_id={getattr(run, 'id')} final_status={getattr(run, 'final_status', None)}" + f"Compliance run completed run_id={run.id} final_status={getattr(run, 'final_status', None)}" ) - except Exception as exc: # noqa: BLE001 - setattr(run, "status", RunStatus.FAILED.value) - setattr(run, "final_status", ComplianceDecision.ERROR.value) - setattr(run, "failure_reason", str(exc)) - setattr(run, "finished_at", datetime.now(timezone.utc)) + except Exception as exc: + run.status = RunStatus.FAILED.value + run.final_status = ComplianceDecision.ERROR.value + run.failure_reason = str(exc) + run.finished_at = datetime.now(UTC) self.repository.save_check_run(run) belief_logger.explore( - f"Compliance run failed run_id={getattr(run, 'id')}: {exc}" + f"Compliance run failed run_id={run.id}: {exc}" ) return ComplianceExecutionResult( diff --git a/backend/src/services/clean_release/compliance_orchestrator.py b/backend/src/services/clean_release/compliance_orchestrator.py index 3584ad60..9fcc725a 100644 --- a/backend/src/services/clean_release/compliance_orchestrator.py +++ b/backend/src/services/clean_release/compliance_orchestrator.py @@ -18,27 +18,20 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import List, Optional +from datetime import UTC, datetime from uuid import uuid4 -from .enums import ( - RunStatus, - ComplianceDecision, - ComplianceStageName, - ViolationCategory, - ViolationSeverity, -) +from ...core.logger import belief_scope, logger from ...models.clean_release import ( ComplianceRun, ComplianceStageRun, - ComplianceViolation, - CheckFinalStatus, ) -from .policy_engine import CleanPolicyEngine +from .enums import ( + ComplianceDecision, + RunStatus, +) from .repository import CleanReleaseRepository from .stages import derive_final_status -from ...core.logger import belief_scope, logger # #region CleanComplianceOrchestrator [TYPE Class] @@ -135,8 +128,8 @@ class CleanComplianceOrchestrator: policy_snapshot_id=policy_id, registry_snapshot_id=registry_snapshot_id, requested_by=actor, - requested_at=datetime.now(timezone.utc), - started_at=datetime.now(timezone.utc), + requested_at=datetime.now(UTC), + started_at=datetime.now(UTC), status=RunStatus.RUNNING, ) logger.reflect( @@ -160,7 +153,7 @@ class CleanComplianceOrchestrator: def execute_stages( self, check_run: ComplianceRun, - forced_results: Optional[List[ComplianceStageRun]] = None, + forced_results: list[ComplianceStageRun] | None = None, ) -> ComplianceRun: with belief_scope("execute_stages"): if forced_results is not None: @@ -196,7 +189,7 @@ class CleanComplianceOrchestrator: if not candidate or not policy or not registry or not manifest: check_run.status = RunStatus.FAILED - check_run.finished_at = datetime.now(timezone.utc) + check_run.finished_at = datetime.now(UTC) return self.repository.save_check_run(check_run) summary = manifest.content_json.get("summary", {}) @@ -207,7 +200,7 @@ class CleanComplianceOrchestrator: else ComplianceDecision.BLOCKED.value ) check_run.status = RunStatus.SUCCEEDED - check_run.finished_at = datetime.now(timezone.utc) + check_run.finished_at = datetime.now(UTC) return self.repository.save_check_run(check_run) @@ -222,7 +215,7 @@ class CleanComplianceOrchestrator: def finalize_run(self, check_run: ComplianceRun) -> ComplianceRun: with belief_scope("finalize_run"): if check_run.status == RunStatus.FAILED: - check_run.finished_at = datetime.now(timezone.utc) + check_run.finished_at = datetime.now(UTC) return self.repository.save_check_run(check_run) if not check_run.final_status: @@ -235,7 +228,7 @@ class CleanComplianceOrchestrator: check_run.final_status = derived.value check_run.status = RunStatus.SUCCEEDED - check_run.finished_at = datetime.now(timezone.utc) + check_run.finished_at = datetime.now(UTC) return self.repository.save_check_run(check_run) # endregion finalize_run diff --git a/backend/src/services/clean_release/demo_data_service.py b/backend/src/services/clean_release/demo_data_service.py index cc33ccef..e1723746 100644 --- a/backend/src/services/clean_release/demo_data_service.py +++ b/backend/src/services/clean_release/demo_data_service.py @@ -45,7 +45,7 @@ def build_namespaced_id(namespace: str, logical_id: str) -> str: def create_isolated_repository(mode: str) -> CleanReleaseRepository: namespace = resolve_namespace(mode) repository = CleanReleaseRepository() - setattr(repository, "namespace", namespace) + repository.namespace = namespace return repository diff --git a/backend/src/services/clean_release/dto.py b/backend/src/services/clean_release/dto.py index 4ae0c58a..3e048ca5 100644 --- a/backend/src/services/clean_release/dto.py +++ b/backend/src/services/clean_release/dto.py @@ -4,16 +4,19 @@ # @RELATION DEPENDS_ON -> pydantic from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import Any + from pydantic import BaseModel, Field -from src.services.clean_release.enums import CandidateStatus, RunStatus, ComplianceDecision + +from src.services.clean_release.enums import CandidateStatus, ComplianceDecision, RunStatus + class CandidateDTO(BaseModel): """DTO for ReleaseCandidate.""" id: str version: str source_snapshot_ref: str - build_id: Optional[str] = None + build_id: str | None = None created_at: datetime created_by: str status: CandidateStatus @@ -25,11 +28,11 @@ class ArtifactDTO(BaseModel): path: str sha256: str size: int - detected_category: Optional[str] = None - declared_category: Optional[str] = None - source_uri: Optional[str] = None - source_host: Optional[str] = None - metadata: Dict[str, Any] = Field(default_factory=dict) + detected_category: str | None = None + declared_category: str | None = None + source_uri: str | None = None + source_host: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) class ManifestDTO(BaseModel): """DTO for DistributionManifest.""" @@ -41,16 +44,16 @@ class ManifestDTO(BaseModel): created_at: datetime created_by: str source_snapshot_ref: str - content_json: Dict[str, Any] + content_json: dict[str, Any] class ComplianceRunDTO(BaseModel): """DTO for ComplianceRun status tracking.""" run_id: str candidate_id: str status: RunStatus - final_status: Optional[ComplianceDecision] = None - report_id: Optional[str] = None - task_id: Optional[str] = None + final_status: ComplianceDecision | None = None + report_id: str | None = None + task_id: str | None = None class ReportDTO(BaseModel): """Compact report view.""" @@ -68,18 +71,18 @@ class CandidateOverviewDTO(BaseModel): version: str source_snapshot_ref: str status: CandidateStatus - latest_manifest_id: Optional[str] = None - latest_manifest_digest: Optional[str] = None - latest_run_id: Optional[str] = None - latest_run_status: Optional[RunStatus] = None - latest_report_id: Optional[str] = None - latest_report_final_status: Optional[ComplianceDecision] = None - latest_policy_snapshot_id: Optional[str] = None - latest_policy_version: Optional[str] = None - latest_registry_snapshot_id: Optional[str] = None - latest_registry_version: Optional[str] = None - latest_approval_decision: Optional[str] = None - latest_publication_id: Optional[str] = None - latest_publication_status: Optional[str] = None + latest_manifest_id: str | None = None + latest_manifest_digest: str | None = None + latest_run_id: str | None = None + latest_run_status: RunStatus | None = None + latest_report_id: str | None = None + latest_report_final_status: ComplianceDecision | None = None + latest_policy_snapshot_id: str | None = None + latest_policy_version: str | None = None + latest_registry_snapshot_id: str | None = None + latest_registry_version: str | None = None + latest_approval_decision: str | None = None + latest_publication_id: str | None = None + latest_publication_status: str | None = None -# #endregion clean_release_dto \ No newline at end of file +# #endregion clean_release_dto diff --git a/backend/src/services/clean_release/enums.py b/backend/src/services/clean_release/enums.py index 25bb55aa..07abc1a7 100644 --- a/backend/src/services/clean_release/enums.py +++ b/backend/src/services/clean_release/enums.py @@ -5,6 +5,7 @@ from enum import Enum + class CandidateStatus(str, Enum): """Lifecycle states for a ReleaseCandidate.""" DRAFT = "DRAFT" @@ -69,4 +70,4 @@ class ViolationCategory(str, Enum): MANIFEST_CONSISTENCY = "MANIFEST_CONSISTENCY" EXTERNAL_ENDPOINT = "EXTERNAL_ENDPOINT" -# #endregion clean_release_enums \ No newline at end of file +# #endregion clean_release_enums diff --git a/backend/src/services/clean_release/exceptions.py b/backend/src/services/clean_release/exceptions.py index f79cb25e..060099a4 100644 --- a/backend/src/services/clean_release/exceptions.py +++ b/backend/src/services/clean_release/exceptions.py @@ -35,4 +35,4 @@ class PublicationGateError(CleanReleaseError): """Raised when publication requirements are not met.""" pass -# #endregion clean_release_exceptions \ No newline at end of file +# #endregion clean_release_exceptions diff --git a/backend/src/services/clean_release/facade.py b/backend/src/services/clean_release/facade.py index f1c569a8..7dca475b 100644 --- a/backend/src/services/clean_release/facade.py +++ b/backend/src/services/clean_release/facade.py @@ -3,20 +3,14 @@ # @LAYER: Application # @RELATION DEPENDS_ON -> ComplianceOrchestrator -from typing import List, Optional -from src.services.clean_release.repositories import ( - CandidateRepository, ArtifactRepository, ManifestRepository, - PolicyRepository, ComplianceRepository, ReportRepository, - ApprovalRepository, PublicationRepository, AuditRepository -) -from src.services.clean_release.dto import ( - CandidateDTO, ArtifactDTO, ManifestDTO, ComplianceRunDTO, - ReportDTO, CandidateOverviewDTO -) -from src.services.clean_release.enums import CandidateStatus, RunStatus, ComplianceDecision -from src.models.clean_release import CleanPolicySnapshot, SourceRegistrySnapshot -from src.core.logger import belief_scope + from src.core.config_manager import ConfigManager +from src.core.logger import belief_scope +from src.models.clean_release import CleanPolicySnapshot, SourceRegistrySnapshot +from src.services.clean_release.dto import CandidateOverviewDTO +from src.services.clean_release.enums import CandidateStatus, ComplianceDecision, RunStatus +from src.services.clean_release.repositories import ApprovalRepository, ArtifactRepository, AuditRepository, CandidateRepository, ComplianceRepository, ManifestRepository, PolicyRepository, PublicationRepository, ReportRepository + class CleanReleaseFacade: """ @@ -46,7 +40,7 @@ class CleanReleaseFacade: self.audit_repo = audit_repo self.config_manager = config_manager - def resolve_active_policy_snapshot(self) -> Optional[CleanPolicySnapshot]: + def resolve_active_policy_snapshot(self) -> CleanPolicySnapshot | None: """ @PURPOSE: Resolve the active policy snapshot based on ConfigManager. """ @@ -57,7 +51,7 @@ class CleanReleaseFacade: return None return self.policy_repo.get_policy_snapshot(policy_id) - def resolve_active_registry_snapshot(self) -> Optional[SourceRegistrySnapshot]: + def resolve_active_registry_snapshot(self) -> SourceRegistrySnapshot | None: """ @PURPOSE: Resolve the active registry snapshot based on ConfigManager. """ @@ -68,7 +62,7 @@ class CleanReleaseFacade: return None return self.policy_repo.get_registry_snapshot(registry_id) - def get_candidate_overview(self, candidate_id: str) -> Optional[CandidateOverviewDTO]: + def get_candidate_overview(self, candidate_id: str) -> CandidateOverviewDTO | None: """ @PURPOSE: Build a comprehensive overview for a candidate. """ @@ -80,14 +74,14 @@ class CleanReleaseFacade: manifest = self.manifest_repo.get_latest_for_candidate(candidate_id) runs = self.compliance_repo.list_runs_by_candidate(candidate_id) latest_run = runs[-1] if runs else None - + report = None if latest_run: report = self.report_repo.get_by_run(latest_run.id) approval = self.approval_repo.get_latest_for_candidate(candidate_id) publication = self.publication_repo.get_latest_for_candidate(candidate_id) - + active_policy = self.resolve_active_policy_snapshot() active_registry = self.resolve_active_registry_snapshot() @@ -111,7 +105,7 @@ class CleanReleaseFacade: latest_publication_status=publication.status if publication else None ) - def list_candidates(self) -> List[CandidateOverviewDTO]: + def list_candidates(self) -> list[CandidateOverviewDTO]: """ @PURPOSE: List all candidates with their current status. """ @@ -119,4 +113,4 @@ class CleanReleaseFacade: candidates = self.candidate_repo.list_all() return [self.get_candidate_overview(c.id) for c in candidates] -# #endregion clean_release_facade \ No newline at end of file +# #endregion clean_release_facade diff --git a/backend/src/services/clean_release/manifest_builder.py b/backend/src/services/clean_release/manifest_builder.py index 24e4a5fc..cfd03ddb 100644 --- a/backend/src/services/clean_release/manifest_builder.py +++ b/backend/src/services/clean_release/manifest_builder.py @@ -8,8 +8,9 @@ from __future__ import annotations import hashlib import json -from datetime import datetime, timezone -from typing import Iterable, List, Dict, Any +from collections.abc import Iterable +from datetime import UTC, datetime +from typing import Any from ...models.clean_release import ( ClassificationType, @@ -20,7 +21,7 @@ from ...models.clean_release import ( def _stable_hash_payload( - candidate_id: str, policy_id: str, items: List[ManifestItem] + candidate_id: str, policy_id: str, items: list[ManifestItem] ) -> str: serialized = [ { @@ -61,7 +62,7 @@ def build_distribution_manifest( candidate_id: str, policy_id: str, generated_by: str, - artifacts: Iterable[Dict[str, Any]], + artifacts: Iterable[dict[str, Any]], ) -> DistributionManifest: items = [ ManifestItem( @@ -98,7 +99,7 @@ def build_distribution_manifest( return DistributionManifest( manifest_id=manifest_id, candidate_id=candidate_id, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), generated_by=generated_by, items=items, summary=summary, @@ -118,7 +119,7 @@ def build_manifest( candidate_id: str, policy_id: str, generated_by: str, - artifacts: Iterable[Dict[str, Any]], + artifacts: Iterable[dict[str, Any]], ) -> DistributionManifest: return build_distribution_manifest( manifest_id=manifest_id, diff --git a/backend/src/services/clean_release/manifest_service.py b/backend/src/services/clean_release/manifest_service.py index 5a6392fb..036da5e5 100644 --- a/backend/src/services/clean_release/manifest_service.py +++ b/backend/src/services/clean_release/manifest_service.py @@ -12,7 +12,7 @@ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any from ...models.clean_release import DistributionManifest from .enums import CandidateStatus @@ -59,7 +59,7 @@ def build_manifest_snapshot( next_version = max((m.manifest_version for m in existing), default=0) + 1 manifest_id = f"manifest-{candidate_id}-v{next_version}" - classified_artifacts: List[Dict[str, Any]] = [ + classified_artifacts: list[dict[str, Any]] = [ { "path": artifact.path, "category": artifact.detected_category or "generic", diff --git a/backend/src/services/clean_release/mappers.py b/backend/src/services/clean_release/mappers.py index edca4b8e..e6941913 100644 --- a/backend/src/services/clean_release/mappers.py +++ b/backend/src/services/clean_release/mappers.py @@ -3,21 +3,10 @@ # @LAYER: Application # @RELATION DEPENDS_ON -> clean_release_dto -from typing import List -from src.models.clean_release import ( - ReleaseCandidate, DistributionManifest, ComplianceRun, - ComplianceStageRun, ComplianceViolation, ComplianceReport, - CleanPolicySnapshot, SourceRegistrySnapshot, ApprovalDecision, - PublicationRecord -) -from src.services.clean_release.dto import ( - CandidateDTO, ArtifactDTO, ManifestDTO, ComplianceRunDTO, - ReportDTO -) -from src.services.clean_release.enums import ( - CandidateStatus, RunStatus, ComplianceDecision, - ViolationSeverity, ViolationCategory -) +from src.models.clean_release import ComplianceReport, ComplianceRun, DistributionManifest, ReleaseCandidate +from src.services.clean_release.dto import CandidateDTO, ComplianceRunDTO, ManifestDTO, ReportDTO +from src.services.clean_release.enums import CandidateStatus, ComplianceDecision, RunStatus + def map_candidate_to_dto(candidate: ReleaseCandidate) -> CandidateDTO: return CandidateDTO( @@ -64,4 +53,4 @@ def map_report_to_dto(report: ComplianceReport) -> ReportDTO: generated_at=report.generated_at ) -# #endregion clean_release_mappers \ No newline at end of file +# #endregion clean_release_mappers diff --git a/backend/src/services/clean_release/policy_engine.py b/backend/src/services/clean_release/policy_engine.py index ace0b52d..4056dda4 100644 --- a/backend/src/services/clean_release/policy_engine.py +++ b/backend/src/services/clean_release/policy_engine.py @@ -11,28 +11,28 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass -from typing import Dict, Iterable, List, Tuple from ...core.logger import belief_scope, logger from ...models.clean_release import ( CleanPolicySnapshot, - SourceRegistrySnapshot, CleanProfilePolicy, ResourceSourceRegistry, + SourceRegistrySnapshot, ) @dataclass class PolicyValidationResult: ok: bool - blocking_reasons: List[str] + blocking_reasons: list[str] @dataclass class SourceValidationResult: ok: bool - violation: Dict | None + violation: dict | None # #region CleanPolicyEngine [TYPE Class] @@ -59,7 +59,7 @@ class CleanPolicyEngine: logger.reason( "Validating enterprise-clean policy and internal registry consistency" ) - reasons: List[str] = [] + reasons: list[str] = [] registry_ref = ( getattr(self.policy, "registry_snapshot_id", None) @@ -123,7 +123,7 @@ class CleanPolicyEngine: ok=len(reasons) == 0, blocking_reasons=reasons ) - def classify_artifact(self, artifact: Dict) -> str: + def classify_artifact(self, artifact: dict) -> str: category = (artifact.get("category") or "").strip() content = dict(getattr(self.policy, "content_json", None) or {}) if not content: @@ -195,14 +195,14 @@ class CleanPolicyEngine: ) def evaluate_candidate( - self, artifacts: Iterable[Dict], sources: Iterable[str] - ) -> Tuple[List[Dict], List[Dict]]: + self, artifacts: Iterable[dict], sources: Iterable[str] + ) -> tuple[list[dict], list[dict]]: with belief_scope("clean_policy_engine.evaluate_candidate"): logger.reason( "Evaluating candidate artifacts and resource sources against enterprise policy" ) - classified: List[Dict] = [] - violations: List[Dict] = [] + classified: list[dict] = [] + violations: list[dict] = [] for artifact in artifacts: classification = self.classify_artifact(artifact) diff --git a/backend/src/services/clean_release/policy_resolution_service.py b/backend/src/services/clean_release/policy_resolution_service.py index e4eec3ae..f08f91c8 100644 --- a/backend/src/services/clean_release/policy_resolution_service.py +++ b/backend/src/services/clean_release/policy_resolution_service.py @@ -12,8 +12,6 @@ from __future__ import annotations -from typing import Optional, Tuple - from ...models.clean_release import CleanPolicySnapshot, SourceRegistrySnapshot from .exceptions import PolicyResolutionError from .repository import CleanReleaseRepository @@ -28,9 +26,9 @@ def resolve_trusted_policy_snapshots( *, config_manager, repository: CleanReleaseRepository, - policy_id_override: Optional[str] = None, - registry_id_override: Optional[str] = None, -) -> Tuple[CleanPolicySnapshot, SourceRegistrySnapshot]: + policy_id_override: str | None = None, + registry_id_override: str | None = None, +) -> tuple[CleanPolicySnapshot, SourceRegistrySnapshot]: if policy_id_override is not None or registry_id_override is not None: raise PolicyResolutionError( "override attempt is forbidden for trusted policy resolution" diff --git a/backend/src/services/clean_release/preparation_service.py b/backend/src/services/clean_release/preparation_service.py index 6b927ea1..d5b447c5 100644 --- a/backend/src/services/clean_release/preparation_service.py +++ b/backend/src/services/clean_release/preparation_service.py @@ -8,23 +8,23 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import Dict, Iterable +from collections.abc import Iterable +from datetime import UTC, datetime +from ...models.clean_release import ReleaseCandidateStatus +from .enums import CandidateStatus from .manifest_builder import build_distribution_manifest from .policy_engine import CleanPolicyEngine from .repository import CleanReleaseRepository -from .enums import CandidateStatus -from ...models.clean_release import ReleaseCandidateStatus def prepare_candidate( repository: CleanReleaseRepository, candidate_id: str, - artifacts: Iterable[Dict], + artifacts: Iterable[dict], sources: Iterable[str], operator_id: str, -) -> Dict: +) -> dict: candidate = repository.get_candidate(candidate_id) if candidate is None: raise ValueError(f"Candidate not found: {candidate_id}") @@ -83,7 +83,7 @@ def prepare_candidate( "status": response_status, "manifest_id": manifest_id_value, "violations": violations, - "prepared_at": datetime.now(timezone.utc).isoformat(), + "prepared_at": datetime.now(UTC).isoformat(), } @@ -94,10 +94,10 @@ def prepare_candidate( def prepare_candidate_legacy( repository: CleanReleaseRepository, candidate_id: str, - artifacts: Iterable[Dict], + artifacts: Iterable[dict], sources: Iterable[str], operator_id: str, -) -> Dict: +) -> dict: return prepare_candidate( repository=repository, candidate_id=candidate_id, diff --git a/backend/src/services/clean_release/publication_service.py b/backend/src/services/clean_release/publication_service.py index 5c54ffb1..5c35490e 100644 --- a/backend/src/services/clean_release/publication_service.py +++ b/backend/src/services/clean_release/publication_service.py @@ -9,8 +9,7 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import List +from datetime import UTC, datetime from uuid import uuid4 from ...core.logger import belief_scope, logger @@ -27,11 +26,11 @@ from .repository import CleanReleaseRepository # @POST: Returns publication list attached to repository. def _get_or_init_publications_store( repository: CleanReleaseRepository, -) -> List[PublicationRecord]: +) -> list[PublicationRecord]: publications = getattr(repository, "publication_records", None) if publications is None: publications = [] - setattr(repository, "publication_records", publications) + repository.publication_records = publications return publications @@ -55,7 +54,7 @@ def _latest_publication_for_candidate( return None return sorted( records, - key=lambda item: item.published_at or datetime.min.replace(tzinfo=timezone.utc), + key=lambda item: item.published_at or datetime.min.replace(tzinfo=UTC), reverse=True, )[0] @@ -76,7 +75,7 @@ def _latest_approval_for_candidate( return None return sorted( scoped, - key=lambda item: item.decided_at or datetime.min.replace(tzinfo=timezone.utc), + key=lambda item: item.decided_at or datetime.min.replace(tzinfo=UTC), reverse=True, )[0] @@ -135,7 +134,7 @@ def publish_candidate( try: candidate.transition_to(CandidateStatus.PUBLISHED) repository.save_candidate(candidate) - except Exception as exc: # noqa: BLE001 + except Exception as exc: logger.explore( f"Candidate transition to PUBLISHED failed candidate_id={candidate_id}: {exc}" ) @@ -146,7 +145,7 @@ def publish_candidate( candidate_id=candidate_id, report_id=report_id, published_by=published_by, - published_at=datetime.now(timezone.utc), + published_at=datetime.now(UTC), target_channel=target_channel, publication_ref=publication_ref, status=PublicationStatus.ACTIVE.value, diff --git a/backend/src/services/clean_release/report_builder.py b/backend/src/services/clean_release/report_builder.py index 34da8258..3ac5965c 100644 --- a/backend/src/services/clean_release/report_builder.py +++ b/backend/src/services/clean_release/report_builder.py @@ -17,12 +17,11 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from uuid import uuid4 -from typing import List -from .enums import RunStatus, ComplianceDecision -from ...models.clean_release import ComplianceRun, ComplianceReport, ComplianceViolation +from ...models.clean_release import ComplianceReport, ComplianceRun, ComplianceViolation +from .enums import ComplianceDecision, RunStatus from .repository import CleanReleaseRepository @@ -31,7 +30,7 @@ class ComplianceReportBuilder: self.repository = repository def build_report_payload( - self, check_run: ComplianceRun, violations: List[ComplianceViolation] + self, check_run: ComplianceRun, violations: list[ComplianceViolation] ) -> ComplianceReport: if check_run.status == RunStatus.RUNNING: raise ValueError("Cannot build report for non-terminal run") @@ -60,7 +59,7 @@ class ComplianceReportBuilder: id=f"CCR-{uuid4()}", run_id=check_run.id, candidate_id=check_run.candidate_id, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), final_status=check_run.final_status, summary_json={ "operator_summary": summary, diff --git a/backend/src/services/clean_release/repositories/__init__.py b/backend/src/services/clean_release/repositories/__init__.py index b08179eb..cc90cf49 100644 --- a/backend/src/services/clean_release/repositories/__init__.py +++ b/backend/src/services/clean_release/repositories/__init__.py @@ -2,27 +2,27 @@ # @BRIEF Export all clean release repositories. # @RELATION DEPENDS_ON -> sqlalchemy -from .candidate_repository import CandidateRepository +from .approval_repository import ApprovalRepository from .artifact_repository import ArtifactRepository +from .audit_repository import AuditRepository, CleanReleaseAuditLog +from .candidate_repository import CandidateRepository +from .compliance_repository import ComplianceRepository from .manifest_repository import ManifestRepository from .policy_repository import PolicyRepository -from .compliance_repository import ComplianceRepository -from .report_repository import ReportRepository -from .approval_repository import ApprovalRepository from .publication_repository import PublicationRepository -from .audit_repository import AuditRepository, CleanReleaseAuditLog +from .report_repository import ReportRepository __all__ = [ - "CandidateRepository", + "ApprovalRepository", "ArtifactRepository", + "AuditRepository", + "CandidateRepository", + "CleanReleaseAuditLog", + "ComplianceRepository", "ManifestRepository", "PolicyRepository", - "ComplianceRepository", - "ReportRepository", - "ApprovalRepository", "PublicationRepository", - "AuditRepository", - "CleanReleaseAuditLog" + "ReportRepository" ] -# #endregion clean_release_repositories \ No newline at end of file +# #endregion clean_release_repositories diff --git a/backend/src/services/clean_release/repositories/approval_repository.py b/backend/src/services/clean_release/repositories/approval_repository.py index 3aa2a225..95ab9b40 100644 --- a/backend/src/services/clean_release/repositories/approval_repository.py +++ b/backend/src/services/clean_release/repositories/approval_repository.py @@ -3,10 +3,12 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import ApprovalDecision + from src.core.logger import belief_scope +from src.models.clean_release import ApprovalDecision + class ApprovalRepository: """ @@ -26,14 +28,14 @@ class ApprovalRepository: self.db.refresh(decision) return decision - def get_by_id(self, decision_id: str) -> Optional[ApprovalDecision]: + def get_by_id(self, decision_id: str) -> ApprovalDecision | None: """ @PURPOSE: Retrieve a decision by ID. """ with belief_scope("ApprovalRepository.get_by_id"): return self.db.query(ApprovalDecision).filter(ApprovalDecision.id == decision_id).first() - def get_latest_for_candidate(self, candidate_id: str) -> Optional[ApprovalDecision]: + def get_latest_for_candidate(self, candidate_id: str) -> ApprovalDecision | None: """ @PURPOSE: Retrieve the latest decision for a candidate. """ @@ -43,11 +45,11 @@ class ApprovalRepository: .order_by(ApprovalDecision.decided_at.desc())\ .first() - def list_by_candidate(self, candidate_id: str) -> List[ApprovalDecision]: + def list_by_candidate(self, candidate_id: str) -> list[ApprovalDecision]: """ @PURPOSE: List all decisions for a specific candidate. """ with belief_scope("ApprovalRepository.list_by_candidate"): return self.db.query(ApprovalDecision).filter(ApprovalDecision.candidate_id == candidate_id).all() -# #endregion approval_repository \ No newline at end of file +# #endregion approval_repository diff --git a/backend/src/services/clean_release/repositories/artifact_repository.py b/backend/src/services/clean_release/repositories/artifact_repository.py index 53a2b2a7..9c31b582 100644 --- a/backend/src/services/clean_release/repositories/artifact_repository.py +++ b/backend/src/services/clean_release/repositories/artifact_repository.py @@ -3,10 +3,12 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import CandidateArtifact + from src.core.logger import belief_scope +from src.models.clean_release import CandidateArtifact + class ArtifactRepository: """ @@ -26,7 +28,7 @@ class ArtifactRepository: self.db.refresh(artifact) return artifact - def save_all(self, artifacts: List[CandidateArtifact]) -> List[CandidateArtifact]: + def save_all(self, artifacts: list[CandidateArtifact]) -> list[CandidateArtifact]: """ @PURPOSE: Persist multiple artifacts in a single transaction. """ @@ -37,18 +39,18 @@ class ArtifactRepository: self.db.refresh(artifact) return artifacts - def get_by_id(self, artifact_id: str) -> Optional[CandidateArtifact]: + def get_by_id(self, artifact_id: str) -> CandidateArtifact | None: """ @PURPOSE: Retrieve an artifact by ID. """ with belief_scope("ArtifactRepository.get_by_id"): return self.db.query(CandidateArtifact).filter(CandidateArtifact.id == artifact_id).first() - def list_by_candidate(self, candidate_id: str) -> List[CandidateArtifact]: + def list_by_candidate(self, candidate_id: str) -> list[CandidateArtifact]: """ @PURPOSE: List all artifacts for a specific candidate. """ with belief_scope("ArtifactRepository.list_by_candidate"): return self.db.query(CandidateArtifact).filter(CandidateArtifact.candidate_id == candidate_id).all() -# #endregion artifact_repository \ No newline at end of file +# #endregion artifact_repository diff --git a/backend/src/services/clean_release/repositories/audit_repository.py b/backend/src/services/clean_release/repositories/audit_repository.py index 0178521b..0acd091d 100644 --- a/backend/src/services/clean_release/repositories/audit_repository.py +++ b/backend/src/services/clean_release/repositories/audit_repository.py @@ -3,16 +3,13 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List -from sqlalchemy.orm import Session -from sqlalchemy import Column, String, DateTime, JSON -from src.models.mapping import Base -from src.core.logger import belief_scope -from datetime import datetime -import uuid +from sqlalchemy.orm import Session + +from src.core.logger import belief_scope from src.models.clean_release import CleanReleaseAuditLog + class AuditRepository: """ @PURPOSE: Encapsulates database operations for CleanReleaseAuditLog. @@ -20,7 +17,7 @@ class AuditRepository: def __init__(self, db: Session): self.db = db - def log(self, action: str, actor: str, candidate_id: Optional[str] = None, details: Optional[dict] = None) -> CleanReleaseAuditLog: + def log(self, action: str, actor: str, candidate_id: str | None = None, details: dict | None = None) -> CleanReleaseAuditLog: """ @PURPOSE: Create an audit log entry. """ @@ -36,11 +33,11 @@ class AuditRepository: self.db.refresh(entry) return entry - def list_by_candidate(self, candidate_id: str) -> List[CleanReleaseAuditLog]: + def list_by_candidate(self, candidate_id: str) -> list[CleanReleaseAuditLog]: """ @PURPOSE: List all audit entries for a specific candidate. """ with belief_scope("AuditRepository.list_by_candidate"): return self.db.query(CleanReleaseAuditLog).filter(CleanReleaseAuditLog.candidate_id == candidate_id).all() -# #endregion audit_repository \ No newline at end of file +# #endregion audit_repository diff --git a/backend/src/services/clean_release/repositories/candidate_repository.py b/backend/src/services/clean_release/repositories/candidate_repository.py index 1a764ea5..25577e30 100644 --- a/backend/src/services/clean_release/repositories/candidate_repository.py +++ b/backend/src/services/clean_release/repositories/candidate_repository.py @@ -3,10 +3,12 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import ReleaseCandidate + from src.core.logger import belief_scope +from src.models.clean_release import ReleaseCandidate + class CandidateRepository: """ @@ -22,7 +24,7 @@ class CandidateRepository: """ with belief_scope("CandidateRepository.save"): # [REASON] Using merge to handle both create and update. - # Note: In a real implementation, we might want to use a separate DB model + # Note: In a real implementation, we might want to use a separate DB model # if the domain model differs significantly from the DB schema. # For now, we assume the domain model is compatible with SQLAlchemy Base if registered. self.db.add(candidate) @@ -30,18 +32,18 @@ class CandidateRepository: self.db.refresh(candidate) return candidate - def get_by_id(self, candidate_id: str) -> Optional[ReleaseCandidate]: + def get_by_id(self, candidate_id: str) -> ReleaseCandidate | None: """ @PURPOSE: Retrieve a candidate by ID. """ with belief_scope("CandidateRepository.get_by_id"): return self.db.query(ReleaseCandidate).filter(ReleaseCandidate.id == candidate_id).first() - def list_all(self) -> List[ReleaseCandidate]: + def list_all(self) -> list[ReleaseCandidate]: """ @PURPOSE: List all candidates. """ with belief_scope("CandidateRepository.list_all"): return self.db.query(ReleaseCandidate).all() -# #endregion candidate_repository \ No newline at end of file +# #endregion candidate_repository diff --git a/backend/src/services/clean_release/repositories/compliance_repository.py b/backend/src/services/clean_release/repositories/compliance_repository.py index ec6f7f82..af3f80a5 100644 --- a/backend/src/services/clean_release/repositories/compliance_repository.py +++ b/backend/src/services/clean_release/repositories/compliance_repository.py @@ -3,10 +3,12 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import ComplianceRun, ComplianceStageRun, ComplianceViolation + from src.core.logger import belief_scope +from src.models.clean_release import ComplianceRun, ComplianceStageRun, ComplianceViolation + class ComplianceRepository: """ @@ -25,14 +27,14 @@ class ComplianceRepository: self.db.refresh(run) return run - def get_run(self, run_id: str) -> Optional[ComplianceRun]: + def get_run(self, run_id: str) -> ComplianceRun | None: """ @PURPOSE: Retrieve a compliance run by ID. """ with belief_scope("ComplianceRepository.get_run"): return self.db.query(ComplianceRun).filter(ComplianceRun.id == run_id).first() - def list_runs_by_candidate(self, candidate_id: str) -> List[ComplianceRun]: + def list_runs_by_candidate(self, candidate_id: str) -> list[ComplianceRun]: """ @PURPOSE: List all runs for a specific candidate. """ @@ -49,7 +51,7 @@ class ComplianceRepository: self.db.refresh(stage_run) return stage_run - def list_stages_by_run(self, run_id: str) -> List[ComplianceStageRun]: + def list_stages_by_run(self, run_id: str) -> list[ComplianceStageRun]: """ @PURPOSE: List all stage runs for a specific compliance run. """ @@ -66,7 +68,7 @@ class ComplianceRepository: self.db.refresh(violation) return violation - def save_violations(self, violations: List[ComplianceViolation]) -> List[ComplianceViolation]: + def save_violations(self, violations: list[ComplianceViolation]) -> list[ComplianceViolation]: """ @PURPOSE: Persist multiple violations. """ @@ -77,11 +79,11 @@ class ComplianceRepository: self.db.refresh(v) return violations - def list_violations_by_run(self, run_id: str) -> List[ComplianceViolation]: + def list_violations_by_run(self, run_id: str) -> list[ComplianceViolation]: """ @PURPOSE: List all violations for a specific compliance run. """ with belief_scope("ComplianceRepository.list_violations_by_run"): return self.db.query(ComplianceViolation).filter(ComplianceViolation.run_id == run_id).all() -# #endregion compliance_repository \ No newline at end of file +# #endregion compliance_repository diff --git a/backend/src/services/clean_release/repositories/manifest_repository.py b/backend/src/services/clean_release/repositories/manifest_repository.py index 62d70828..e3aad435 100644 --- a/backend/src/services/clean_release/repositories/manifest_repository.py +++ b/backend/src/services/clean_release/repositories/manifest_repository.py @@ -5,10 +5,11 @@ # @RELATION DEPENDS_ON -> sqlalchemy # @RELATION DEPENDS_ON -> belief_scope -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import DistributionManifest + from src.core.logger import belief_scope +from src.models.clean_release import DistributionManifest # #region ManifestRepository [C:3] [TYPE Class] @@ -45,7 +46,7 @@ class ManifestRepository: # @PRE: manifest_id is a valid string identifier. # @POST: Returns DistributionManifest if found, None otherwise. # @RELATION: DEPENDS_ON -> DistributionManifest - def get_by_id(self, manifest_id: str) -> Optional[DistributionManifest]: + def get_by_id(self, manifest_id: str) -> DistributionManifest | None: with belief_scope("ManifestRepository.get_by_id"): return self.db.query(DistributionManifest).filter( DistributionManifest.id == manifest_id @@ -57,7 +58,7 @@ class ManifestRepository: # @PRE: candidate_id is a valid string identifier. # @POST: Returns the highest manifest_version manifest for the candidate, or None. # @RELATION: DEPENDS_ON -> DistributionManifest - def get_latest_for_candidate(self, candidate_id: str) -> Optional[DistributionManifest]: + def get_latest_for_candidate(self, candidate_id: str) -> DistributionManifest | None: with belief_scope("ManifestRepository.get_latest_for_candidate"): return ( self.db.query(DistributionManifest) @@ -72,7 +73,7 @@ class ManifestRepository: # @PRE: candidate_id is a valid string identifier. # @POST: Returns a list of DistributionManifest instances (may be empty). # @RELATION: DEPENDS_ON -> DistributionManifest - def list_by_candidate(self, candidate_id: str) -> List[DistributionManifest]: + def list_by_candidate(self, candidate_id: str) -> list[DistributionManifest]: with belief_scope("ManifestRepository.list_by_candidate"): return ( self.db.query(DistributionManifest) @@ -83,4 +84,4 @@ class ManifestRepository: # #endregion ManifestRepository -# #endregion ManifestRepositoryModule \ No newline at end of file +# #endregion ManifestRepositoryModule diff --git a/backend/src/services/clean_release/repositories/policy_repository.py b/backend/src/services/clean_release/repositories/policy_repository.py index c7cb0f68..4dc26c5f 100644 --- a/backend/src/services/clean_release/repositories/policy_repository.py +++ b/backend/src/services/clean_release/repositories/policy_repository.py @@ -3,10 +3,11 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List from sqlalchemy.orm import Session -from src.models.clean_release import CleanPolicySnapshot, SourceRegistrySnapshot + from src.core.logger import belief_scope +from src.models.clean_release import CleanPolicySnapshot, SourceRegistrySnapshot + class PolicyRepository: """ @@ -25,7 +26,7 @@ class PolicyRepository: self.db.refresh(snapshot) return snapshot - def get_policy_snapshot(self, snapshot_id: str) -> Optional[CleanPolicySnapshot]: + def get_policy_snapshot(self, snapshot_id: str) -> CleanPolicySnapshot | None: """ @PURPOSE: Retrieve a policy snapshot by ID. """ @@ -42,11 +43,11 @@ class PolicyRepository: self.db.refresh(snapshot) return snapshot - def get_registry_snapshot(self, snapshot_id: str) -> Optional[SourceRegistrySnapshot]: + def get_registry_snapshot(self, snapshot_id: str) -> SourceRegistrySnapshot | None: """ @PURPOSE: Retrieve a registry snapshot by ID. """ with belief_scope("PolicyRepository.get_registry_snapshot"): return self.db.query(SourceRegistrySnapshot).filter(SourceRegistrySnapshot.id == snapshot_id).first() -# #endregion policy_repository \ No newline at end of file +# #endregion policy_repository diff --git a/backend/src/services/clean_release/repositories/publication_repository.py b/backend/src/services/clean_release/repositories/publication_repository.py index f6bf28e4..f006e7fb 100644 --- a/backend/src/services/clean_release/repositories/publication_repository.py +++ b/backend/src/services/clean_release/repositories/publication_repository.py @@ -3,10 +3,12 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import PublicationRecord + from src.core.logger import belief_scope +from src.models.clean_release import PublicationRecord + class PublicationRepository: """ @@ -26,14 +28,14 @@ class PublicationRepository: self.db.refresh(record) return record - def get_by_id(self, record_id: str) -> Optional[PublicationRecord]: + def get_by_id(self, record_id: str) -> PublicationRecord | None: """ @PURPOSE: Retrieve a record by ID. """ with belief_scope("PublicationRepository.get_by_id"): return self.db.query(PublicationRecord).filter(PublicationRecord.id == record_id).first() - def get_latest_for_candidate(self, candidate_id: str) -> Optional[PublicationRecord]: + def get_latest_for_candidate(self, candidate_id: str) -> PublicationRecord | None: """ @PURPOSE: Retrieve the latest record for a candidate. """ @@ -43,11 +45,11 @@ class PublicationRepository: .order_by(PublicationRecord.published_at.desc())\ .first() - def list_by_candidate(self, candidate_id: str) -> List[PublicationRecord]: + def list_by_candidate(self, candidate_id: str) -> list[PublicationRecord]: """ @PURPOSE: List all records for a specific candidate. """ with belief_scope("PublicationRepository.list_by_candidate"): return self.db.query(PublicationRecord).filter(PublicationRecord.candidate_id == candidate_id).all() -# #endregion publication_repository \ No newline at end of file +# #endregion publication_repository diff --git a/backend/src/services/clean_release/repositories/report_repository.py b/backend/src/services/clean_release/repositories/report_repository.py index 2b980bcc..da36c0ba 100644 --- a/backend/src/services/clean_release/repositories/report_repository.py +++ b/backend/src/services/clean_release/repositories/report_repository.py @@ -3,10 +3,12 @@ # @LAYER: Infra # @RELATION DEPENDS_ON -> sqlalchemy -from typing import Optional, List + from sqlalchemy.orm import Session -from src.models.clean_release import ComplianceReport + from src.core.logger import belief_scope +from src.models.clean_release import ComplianceReport + class ReportRepository: """ @@ -26,25 +28,25 @@ class ReportRepository: self.db.refresh(report) return report - def get_by_id(self, report_id: str) -> Optional[ComplianceReport]: + def get_by_id(self, report_id: str) -> ComplianceReport | None: """ @PURPOSE: Retrieve a report by ID. """ with belief_scope("ReportRepository.get_by_id"): return self.db.query(ComplianceReport).filter(ComplianceReport.id == report_id).first() - def get_by_run(self, run_id: str) -> Optional[ComplianceReport]: + def get_by_run(self, run_id: str) -> ComplianceReport | None: """ @PURPOSE: Retrieve a report for a specific compliance run. """ with belief_scope("ReportRepository.get_by_run"): return self.db.query(ComplianceReport).filter(ComplianceReport.run_id == run_id).first() - def list_by_candidate(self, candidate_id: str) -> List[ComplianceReport]: + def list_by_candidate(self, candidate_id: str) -> list[ComplianceReport]: """ @PURPOSE: List all reports for a specific candidate. """ with belief_scope("ReportRepository.list_by_candidate"): return self.db.query(ComplianceReport).filter(ComplianceReport.candidate_id == candidate_id).all() -# #endregion report_repository \ No newline at end of file +# #endregion report_repository diff --git a/backend/src/services/clean_release/repository.py b/backend/src/services/clean_release/repository.py index a489958d..960063a4 100644 --- a/backend/src/services/clean_release/repository.py +++ b/backend/src/services/clean_release/repository.py @@ -7,12 +7,12 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, cast +from typing import Any, cast from ...models.clean_release import ( CleanPolicySnapshot, - ComplianceRun, ComplianceReport, + ComplianceRun, ComplianceStageRun, ComplianceViolation, DistributionManifest, @@ -25,64 +25,64 @@ from ...models.clean_release import ( # @BRIEF Data access object for clean release lifecycle. @dataclass class CleanReleaseRepository: - candidates: Dict[str, ReleaseCandidate] = field(default_factory=dict) - policies: Dict[str, CleanPolicySnapshot] = field(default_factory=dict) - registries: Dict[str, SourceRegistrySnapshot] = field(default_factory=dict) - artifacts: Dict[str, object] = field(default_factory=dict) - manifests: Dict[str, DistributionManifest] = field(default_factory=dict) - check_runs: Dict[str, ComplianceRun] = field(default_factory=dict) - stage_runs: Dict[str, ComplianceStageRun] = field(default_factory=dict) - reports: Dict[str, ComplianceReport] = field(default_factory=dict) - violations: Dict[str, ComplianceViolation] = field(default_factory=dict) - audit_events: List[Dict[str, Any]] = field(default_factory=list) + candidates: dict[str, ReleaseCandidate] = field(default_factory=dict) + policies: dict[str, CleanPolicySnapshot] = field(default_factory=dict) + registries: dict[str, SourceRegistrySnapshot] = field(default_factory=dict) + artifacts: dict[str, object] = field(default_factory=dict) + manifests: dict[str, DistributionManifest] = field(default_factory=dict) + check_runs: dict[str, ComplianceRun] = field(default_factory=dict) + stage_runs: dict[str, ComplianceStageRun] = field(default_factory=dict) + reports: dict[str, ComplianceReport] = field(default_factory=dict) + violations: dict[str, ComplianceViolation] = field(default_factory=dict) + audit_events: list[dict[str, Any]] = field(default_factory=list) def _entity_id(self, entity: Any) -> str: - return cast(str, str(getattr(cast(Any, entity), "id"))) + return cast(str, str(cast(Any, entity).id)) def _candidate_ref(self, entity: Any) -> str: - return cast(str, str(getattr(cast(Any, entity), "candidate_id"))) + return cast(str, str(cast(Any, entity).candidate_id)) def _run_ref(self, entity: Any) -> str: - return cast(str, str(getattr(cast(Any, entity), "run_id"))) + return cast(str, str(cast(Any, entity).run_id)) def save_candidate(self, candidate: ReleaseCandidate) -> ReleaseCandidate: - cast(Dict[str, ReleaseCandidate], self.candidates)[ + cast(dict[str, ReleaseCandidate], self.candidates)[ self._entity_id(candidate) ] = candidate return candidate - def get_candidate(self, candidate_id: str) -> Optional[ReleaseCandidate]: + def get_candidate(self, candidate_id: str) -> ReleaseCandidate | None: return self.candidates.get(candidate_id) def save_policy(self, policy: CleanPolicySnapshot) -> CleanPolicySnapshot: - cast(Dict[str, CleanPolicySnapshot], self.policies)[self._entity_id(policy)] = ( + cast(dict[str, CleanPolicySnapshot], self.policies)[self._entity_id(policy)] = ( policy ) return policy - def get_policy(self, policy_id: str) -> Optional[CleanPolicySnapshot]: + def get_policy(self, policy_id: str) -> CleanPolicySnapshot | None: return self.policies.get(policy_id) - def get_active_policy(self) -> Optional[CleanPolicySnapshot]: + def get_active_policy(self) -> CleanPolicySnapshot | None: # In-memory repo doesn't track 'active' flag on snapshot, # this should be resolved by facade using ConfigManager. return next(iter(self.policies.values()), None) def save_registry(self, registry: SourceRegistrySnapshot) -> SourceRegistrySnapshot: - cast(Dict[str, SourceRegistrySnapshot], self.registries)[ + cast(dict[str, SourceRegistrySnapshot], self.registries)[ self._entity_id(registry) ] = registry return registry - def get_registry(self, registry_id: str) -> Optional[SourceRegistrySnapshot]: + def get_registry(self, registry_id: str) -> SourceRegistrySnapshot | None: return self.registries.get(registry_id) def save_artifact(self, artifact) -> object: - cast(Dict[str, object], self.artifacts)[self._entity_id(artifact)] = artifact + cast(dict[str, object], self.artifacts)[self._entity_id(artifact)] = artifact return artifact - def get_artifacts_by_candidate(self, candidate_id: str) -> List[object]: - artifacts = cast(List[Any], list(self.artifacts.values())) + def get_artifacts_by_candidate(self, candidate_id: str) -> list[object]: + artifacts = cast(list[Any], list(self.artifacts.values())) return [ artifact for artifact in artifacts @@ -90,12 +90,12 @@ class CleanReleaseRepository: ] def save_manifest(self, manifest: DistributionManifest) -> DistributionManifest: - cast(Dict[str, DistributionManifest], self.manifests)[ + cast(dict[str, DistributionManifest], self.manifests)[ self._entity_id(manifest) ] = manifest return manifest - def get_manifest(self, manifest_id: str) -> Optional[DistributionManifest]: + def get_manifest(self, manifest_id: str) -> DistributionManifest | None: return self.manifests.get(manifest_id) def save_distribution_manifest( @@ -105,33 +105,33 @@ class CleanReleaseRepository: def get_distribution_manifest( self, manifest_id: str - ) -> Optional[DistributionManifest]: + ) -> DistributionManifest | None: return self.get_manifest(manifest_id) def save_check_run(self, check_run: ComplianceRun) -> ComplianceRun: - cast(Dict[str, ComplianceRun], self.check_runs)[self._entity_id(check_run)] = ( + cast(dict[str, ComplianceRun], self.check_runs)[self._entity_id(check_run)] = ( check_run ) return check_run def save_stage_run(self, stage_run: ComplianceStageRun) -> ComplianceStageRun: - cast(Dict[str, ComplianceStageRun], self.stage_runs)[ + cast(dict[str, ComplianceStageRun], self.stage_runs)[ self._entity_id(stage_run) ] = stage_run return stage_run - def get_check_run(self, check_run_id: str) -> Optional[ComplianceRun]: + def get_check_run(self, check_run_id: str) -> ComplianceRun | None: return self.check_runs.get(check_run_id) def save_compliance_run(self, run: ComplianceRun) -> ComplianceRun: return self.save_check_run(run) - def get_compliance_run(self, run_id: str) -> Optional[ComplianceRun]: + def get_compliance_run(self, run_id: str) -> ComplianceRun | None: return self.get_check_run(run_id) def save_report(self, report: ComplianceReport) -> ComplianceReport: report_id = self._entity_id(report) - store = cast(Dict[str, ComplianceReport], self.reports) + store = cast(dict[str, ComplianceReport], self.reports) existing = store.get(report_id) if existing is not None: raise ValueError( @@ -140,32 +140,32 @@ class CleanReleaseRepository: store[report_id] = report return report - def get_report(self, report_id: str) -> Optional[ComplianceReport]: + def get_report(self, report_id: str) -> ComplianceReport | None: return self.reports.get(report_id) def save_violation(self, violation: ComplianceViolation) -> ComplianceViolation: - cast(Dict[str, ComplianceViolation], self.violations)[ + cast(dict[str, ComplianceViolation], self.violations)[ self._entity_id(violation) ] = violation return violation - def get_violations_by_run(self, run_id: str) -> List[ComplianceViolation]: - violations = cast(List[ComplianceViolation], list(self.violations.values())) + def get_violations_by_run(self, run_id: str) -> list[ComplianceViolation]: + violations = cast(list[ComplianceViolation], list(self.violations.values())) return [ violation for violation in violations if self._run_ref(violation) == run_id ] def get_manifests_by_candidate( self, candidate_id: str - ) -> List[DistributionManifest]: - manifests = cast(List[DistributionManifest], list(self.manifests.values())) + ) -> list[DistributionManifest]: + manifests = cast(list[DistributionManifest], list(self.manifests.values())) return [ manifest for manifest in manifests if self._candidate_ref(manifest) == candidate_id ] - def append_audit_event(self, payload: Dict[str, Any]) -> None: + def append_audit_event(self, payload: dict[str, Any]) -> None: self.audit_events.append(payload) def clear_history(self) -> None: diff --git a/backend/src/services/clean_release/source_isolation.py b/backend/src/services/clean_release/source_isolation.py index 1c4b0d30..3708b6f6 100644 --- a/backend/src/services/clean_release/source_isolation.py +++ b/backend/src/services/clean_release/source_isolation.py @@ -6,18 +6,18 @@ from __future__ import annotations -from typing import Dict, Iterable, List +from collections.abc import Iterable from ...models.clean_release import ResourceSourceRegistry def validate_internal_sources( registry: ResourceSourceRegistry, endpoints: Iterable[str] -) -> Dict: +) -> dict: allowed_hosts = { entry.host.strip().lower() for entry in registry.entries if entry.enabled } - violations: List[Dict] = [] + violations: list[dict] = [] for endpoint in endpoints: normalized = (endpoint or "").strip().lower() diff --git a/backend/src/services/clean_release/stages/__init__.py b/backend/src/services/clean_release/stages/__init__.py index 2b829a1a..245334a7 100644 --- a/backend/src/services/clean_release/stages/__init__.py +++ b/backend/src/services/clean_release/stages/__init__.py @@ -7,22 +7,23 @@ from __future__ import annotations -from typing import Dict, Iterable, List +from collections.abc import Iterable +from typing import Dict, List -from ..enums import ComplianceDecision, ComplianceStageName from ....models.clean_release import ( - ComplianceStageRun, CheckFinalStatus, CheckStageResult, CheckStageStatus, + ComplianceStageRun, ) +from ..enums import ComplianceDecision, ComplianceStageName from .base import ComplianceStage from .data_purity import DataPurityStage from .internal_sources_only import InternalSourcesOnlyStage from .manifest_consistency import ManifestConsistencyStage from .no_external_endpoints import NoExternalEndpointsStage -MANDATORY_STAGE_ORDER: List[ComplianceStageName] = [ +MANDATORY_STAGE_ORDER: list[ComplianceStageName] = [ ComplianceStageName.DATA_PURITY, ComplianceStageName.INTERNAL_SOURCES_ONLY, ComplianceStageName.NO_EXTERNAL_ENDPOINTS, @@ -34,7 +35,7 @@ MANDATORY_STAGE_ORDER: List[ComplianceStageName] = [ # @BRIEF Build default deterministic stage pipeline implementation order. # @PRE: None. # @POST: Returns stage instances in mandatory execution order. -def build_default_stages() -> List[ComplianceStage]: +def build_default_stages() -> list[ComplianceStage]: return [ DataPurityStage(), InternalSourcesOnlyStage(), @@ -52,8 +53,8 @@ def build_default_stages() -> List[ComplianceStage]: # @POST: Returns stage->status dictionary for downstream evaluation. def stage_result_map( stage_results: Iterable[ComplianceStageRun | CheckStageResult], -) -> Dict[ComplianceStageName, CheckStageStatus]: - normalized: Dict[ComplianceStageName, CheckStageStatus] = {} +) -> dict[ComplianceStageName, CheckStageStatus]: + normalized: dict[ComplianceStageName, CheckStageStatus] = {} for result in stage_results: if isinstance(result, CheckStageResult): normalized[ComplianceStageName(result.stage.value)] = CheckStageStatus( @@ -90,8 +91,8 @@ def stage_result_map( # @PRE: stage_status_map contains zero or more known stage statuses. # @POST: Returns ordered list of missing mandatory stages. def missing_mandatory_stages( - stage_status_map: Dict[ComplianceStageName, CheckStageStatus], -) -> List[ComplianceStageName]: + stage_status_map: dict[ComplianceStageName, CheckStageStatus], +) -> list[ComplianceStageName]: return [stage for stage in MANDATORY_STAGE_ORDER if stage not in stage_status_map] diff --git a/backend/src/services/clean_release/stages/base.py b/backend/src/services/clean_release/stages/base.py index 16fcf73c..b7e189d5 100644 --- a/backend/src/services/clean_release/stages/base.py +++ b/backend/src/services/clean_release/stages/base.py @@ -8,8 +8,8 @@ from __future__ import annotations from dataclasses import dataclass, field -from datetime import datetime, timezone -from typing import Any, Dict, List, Protocol +from datetime import UTC, datetime +from typing import Any, Protocol from uuid import uuid4 from ....core.logger import belief_scope, logger @@ -45,8 +45,8 @@ class ComplianceStageContext: @dataclass class StageExecutionResult: decision: ComplianceDecision - details_json: Dict[str, Any] = field(default_factory=dict) - violations: List[ComplianceViolation] = field(default_factory=list) + details_json: dict[str, Any] = field(default_factory=dict) + violations: list[ComplianceViolation] = field(default_factory=list) # #endregion StageExecutionResult @@ -76,7 +76,7 @@ def build_stage_run_record( finished_at: datetime | None = None, ) -> ComplianceStageRun: with belief_scope("build_stage_run_record"): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) return ComplianceStageRun( id=f"stg-{uuid4()}", run_id=run_id, @@ -106,7 +106,7 @@ def build_violation( message: str, artifact_path: str | None = None, severity: ViolationSeverity = ViolationSeverity.MAJOR, - evidence_json: Dict[str, Any] | None = None, + evidence_json: dict[str, Any] | None = None, blocked_release: bool = True, ) -> ComplianceViolation: with belief_scope("build_violation"): diff --git a/backend/src/services/dataset_review/__init__.py b/backend/src/services/dataset_review/__init__.py index 346ffa61..db488d7c 100644 --- a/backend/src/services/dataset_review/__init__.py +++ b/backend/src/services/dataset_review/__init__.py @@ -4,4 +4,4 @@ # @RELATION EXPORTS -> [DatasetReviewOrchestrator:Class] # @LAYER: Services # -# #endregion dataset_review \ No newline at end of file +# #endregion dataset_review diff --git a/backend/src/services/dataset_review/clarification_engine.py b/backend/src/services/dataset_review/clarification_engine.py index 3ba3369d..9575b719 100644 --- a/backend/src/services/dataset_review/clarification_engine.py +++ b/backend/src/services/dataset_review/clarification_engine.py @@ -19,7 +19,6 @@ from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime -from typing import List, Optional from src.core.logger import belief_scope, logger from src.models.auth import User @@ -36,18 +35,18 @@ from src.models.dataset_review import ( SessionPhase, ValidationFinding, ) -from src.services.dataset_review.repositories.session_repository import ( - DatasetReviewSessionRepository, -) from src.services.dataset_review.clarification_pkg._helpers import ( - select_next_open_question, - count_resolved_questions, - count_remaining_questions, - normalize_answer_value, build_impact_summary, - upsert_clarification_finding, + count_remaining_questions, + count_resolved_questions, derive_readiness_state, derive_recommended_action, + normalize_answer_value, + select_next_open_question, + upsert_clarification_finding, +) +from src.services.dataset_review.repositories.session_repository import ( + DatasetReviewSessionRepository, ) @@ -60,7 +59,7 @@ class ClarificationQuestionPayload: topic_ref: str question_text: str why_it_matters: str - current_guess: Optional[str] + current_guess: str | None priority: int state: QuestionState options: list[dict[str, object]] = field(default_factory=list) @@ -74,9 +73,9 @@ class ClarificationQuestionPayload: @dataclass class ClarificationStateResult: clarification_session: ClarificationSession - current_question: Optional[ClarificationQuestionPayload] + current_question: ClarificationQuestionPayload | None session: DatasetReviewSession - changed_findings: List[ValidationFinding] = field(default_factory=list) + changed_findings: list[ValidationFinding] = field(default_factory=list) # #endregion ClarificationStateResult @@ -89,7 +88,7 @@ class ClarificationAnswerCommand: session: DatasetReviewSession question_id: str answer_kind: AnswerKind - answer_value: Optional[str] + answer_value: str | None user: User @@ -118,7 +117,7 @@ class ClarificationEngine: # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence. def build_question_payload( self, session: DatasetReviewSession, - ) -> Optional[ClarificationQuestionPayload]: + ) -> ClarificationQuestionPayload | None: with belief_scope("ClarificationEngine.build_question_payload"): clarification_session = self._get_latest_clarification_session(session) if clarification_session is None: @@ -214,9 +213,7 @@ class ClarificationEngine: answer_value=normalized_answer_value, db_session=self.repository.db, ) - if command.answer_kind == AnswerKind.SELECTED: - question.state = QuestionState.ANSWERED - elif command.answer_kind == AnswerKind.CUSTOM: + if command.answer_kind == AnswerKind.SELECTED or command.answer_kind == AnswerKind.CUSTOM: question.state = QuestionState.ANSWERED elif command.answer_kind == AnswerKind.SKIPPED: question.state = QuestionState.SKIPPED @@ -267,7 +264,7 @@ class ClarificationEngine: # region _get_latest_clarification_session [TYPE Function] # @PURPOSE: Select the latest clarification session for the current dataset review aggregate. - def _get_latest_clarification_session(self, session: DatasetReviewSession) -> Optional[ClarificationSession]: + def _get_latest_clarification_session(self, session: DatasetReviewSession) -> ClarificationSession | None: if not session.clarification_sessions: return None ordered = sorted(session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True) @@ -277,7 +274,7 @@ class ClarificationEngine: # region _find_question [TYPE Function] # @PURPOSE: Resolve a clarification question from the active clarification aggregate. - def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> Optional[ClarificationQuestion]: + def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> ClarificationQuestion | None: for q in clarification_session.questions: if q.question_id == question_id: return q diff --git a/backend/src/services/dataset_review/clarification_pkg/_helpers.py b/backend/src/services/dataset_review/clarification_pkg/_helpers.py index e8712ecf..86c1e7bb 100644 --- a/backend/src/services/dataset_review/clarification_pkg/_helpers.py +++ b/backend/src/services/dataset_review/clarification_pkg/_helpers.py @@ -7,11 +7,9 @@ from __future__ import annotations import uuid from datetime import datetime -from typing import List, Optional from src.models.dataset_review import ( AnswerKind, - ClarificationAnswer, ClarificationQuestion, ClarificationSession, DatasetReviewSession, @@ -29,7 +27,7 @@ from src.models.dataset_review import ( # @BRIEF Select the next unresolved question in deterministic priority order. def select_next_open_question( clarification_session: ClarificationSession, -) -> Optional[ClarificationQuestion]: +) -> ClarificationQuestion | None: open_questions = [ q for q in clarification_session.questions if q.state == QuestionState.OPEN ] @@ -68,9 +66,9 @@ def count_remaining_questions(clarification_session: ClarificationSession) -> in # @BRIEF Validate and normalize answer payload based on answer kind and active question options. def normalize_answer_value( answer_kind: AnswerKind, - answer_value: Optional[str], + answer_value: str | None, question: ClarificationQuestion, -) -> Optional[str]: +) -> str | None: normalized = str(answer_value).strip() if answer_value is not None else None if answer_kind in {AnswerKind.SELECTED, AnswerKind.CUSTOM} and not normalized: raise ValueError("answer_value is required for selected or custom clarification answers") @@ -93,7 +91,7 @@ def normalize_answer_value( def build_impact_summary( question: ClarificationQuestion, answer_kind: AnswerKind, - answer_value: Optional[str], + answer_value: str | None, ) -> str: if answer_kind == AnswerKind.SKIPPED: return f"Clarification for {question.topic_ref} was skipped and remains unresolved." @@ -112,9 +110,9 @@ def upsert_clarification_finding( session: DatasetReviewSession, question: ClarificationQuestion, answer_kind: AnswerKind, - answer_value: Optional[str], + answer_value: str | None, db_session, -) -> Optional[ValidationFinding]: +) -> ValidationFinding | None: caused_by_ref = f"clarification:{question.question_id}" existing = next( (f for f in session.findings if f.area == FindingArea.CLARIFICATION and f.caused_by_ref == caused_by_ref), @@ -176,7 +174,7 @@ def upsert_clarification_finding( # @BRIEF Recompute readiness after clarification mutation while preserving unresolved visibility semantics. def derive_readiness_state( session: DatasetReviewSession, - clarification_session: Optional[ClarificationSession], + clarification_session: ClarificationSession | None, ) -> ReadinessState: if clarification_session is None: return session.readiness_state @@ -194,7 +192,7 @@ def derive_readiness_state( # @BRIEF Recompute next-action guidance after clarification mutations. def derive_recommended_action( session: DatasetReviewSession, - clarification_session: Optional[ClarificationSession], + clarification_session: ClarificationSession | None, ) -> RecommendedAction: if clarification_session is None: return session.recommended_action diff --git a/backend/src/services/dataset_review/event_logger.py b/backend/src/services/dataset_review/event_logger.py index dca1ae69..0233fe5e 100644 --- a/backend/src/services/dataset_review/event_logger.py +++ b/backend/src/services/dataset_review/event_logger.py @@ -12,12 +12,13 @@ from __future__ import annotations # #region SessionEventLoggerImports [TYPE Block] from dataclasses import dataclass, field -from typing import Any, Dict, Optional +from typing import Any from sqlalchemy.orm import Session from src.core.logger import belief_scope, logger from src.models.dataset_review import DatasetReviewSession, SessionEvent + # #endregion SessionEventLoggerImports @@ -29,9 +30,9 @@ class SessionEventPayload: actor_user_id: str event_type: str event_summary: str - current_phase: Optional[str] = None - readiness_state: Optional[str] = None - event_details: Dict[str, Any] = field(default_factory=dict) + current_phase: str | None = None + readiness_state: str | None = None + event_details: dict[str, Any] = field(default_factory=dict) # #endregion SessionEventPayload @@ -132,7 +133,7 @@ class SessionEventLogger: actor_user_id: str, event_type: str, event_summary: str, - event_details: Optional[Dict[str, Any]] = None, + event_details: dict[str, Any] | None = None, ) -> SessionEvent: return self.log_event( SessionEventPayload( @@ -148,4 +149,4 @@ class SessionEventLogger: # endregion log_for_session # #endregion SessionEventLogger -# #endregion SessionEventLoggerModule \ No newline at end of file +# #endregion SessionEventLoggerModule diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index f2d274aa..3fc4278e 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -18,9 +18,8 @@ from __future__ import annotations -from dataclasses import dataclass, field from datetime import datetime -from typing import Any, Dict, List, Optional, cast +from typing import Any, cast from src.core.config_manager import ConfigManager from src.core.logger import belief_scope, logger @@ -34,13 +33,8 @@ from src.core.utils.superset_context_extractor import ( SupersetContextExtractor, SupersetParsedContext, ) -from src.models.auth import User from src.models.dataset_review import ( ApprovalState, - BusinessSummarySource, - CompiledPreview, - ConfidenceState, - DatasetProfile, DatasetReviewSession, DatasetRunContext, ExecutionMapping, @@ -54,8 +48,8 @@ from src.models.dataset_review import ( MappingMethod, MappingStatus, PreviewStatus, - RecommendedAction, ReadinessState, + RecommendedAction, ResolutionState, SessionPhase, SessionStatus, @@ -63,29 +57,27 @@ from src.models.dataset_review import ( ValidationFinding, VariableKind, ) +from src.services.dataset_review.event_logger import SessionEventPayload +from src.services.dataset_review.orchestrator_pkg._commands import ( + LaunchDatasetCommand, + LaunchDatasetResult, + PreparePreviewCommand, + PreparePreviewResult, + StartSessionCommand, + StartSessionResult, +) +from src.services.dataset_review.orchestrator_pkg._helpers import ( + build_execution_snapshot, + build_initial_profile, + build_launch_blockers, + build_partial_recovery_findings, + get_latest_preview, + parse_dataset_selection, +) from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionRepository, ) from src.services.dataset_review.semantic_resolver import SemanticSourceResolver -from src.services.dataset_review.event_logger import SessionEventPayload -from src.services.dataset_review.orchestrator_pkg._commands import ( - StartSessionCommand, - StartSessionResult, - PreparePreviewCommand, - PreparePreviewResult, - LaunchDatasetCommand, - LaunchDatasetResult, -) -from src.services.dataset_review.orchestrator_pkg._helpers import ( - parse_dataset_selection, - build_initial_profile, - build_partial_recovery_findings, - build_execution_snapshot, - build_launch_blockers, - get_latest_preview, - compute_preview_fingerprint, - extract_effective_filter_value, -) logger = cast(Any, logger) @@ -112,8 +104,8 @@ class DatasetReviewOrchestrator: self, repository: DatasetReviewSessionRepository, config_manager: ConfigManager, - task_manager: Optional[TaskManager] = None, - semantic_resolver: Optional[SemanticSourceResolver] = None, + task_manager: TaskManager | None = None, + semantic_resolver: SemanticSourceResolver | None = None, ) -> None: self.repository = repository self.config_manager = config_manager @@ -152,11 +144,11 @@ class DatasetReviewOrchestrator: logger.reason("Starting dataset review session", extra={"user_id": command.user.id, "environment_id": normalized_environment_id, "source_kind": normalized_source_kind}) - parsed_context: Optional[SupersetParsedContext] = None - findings: List[ValidationFinding] = [] + parsed_context: SupersetParsedContext | None = None + findings: list[ValidationFinding] = [] dataset_ref = normalized_source_input - dataset_id: Optional[int] = None - dashboard_id: Optional[int] = None + dataset_id: int | None = None + dashboard_id: int | None = None readiness_state = ReadinessState.IMPORTING recommended_action = RecommendedAction.REVIEW_DOCUMENTATION current_phase = SessionPhase.RECOVERY @@ -194,9 +186,9 @@ class DatasetReviewOrchestrator: ) persisted_session = cast(Any, self.repository.create_session(session)) - recovered_filters: List[ImportedFilter] = [] - template_variables: List[TemplateVariable] = [] - execution_mappings: List[ExecutionMapping] = [] + recovered_filters: list[ImportedFilter] = [] + template_variables: list[TemplateVariable] = [] + execution_mappings: list[ExecutionMapping] = [] if normalized_source_kind == "superset_link" and parsed_context is not None: recovered_filters, template_variables, execution_mappings, findings = ( self._build_recovery_bootstrap( @@ -464,8 +456,8 @@ class DatasetReviewOrchestrator: environment, session: DatasetReviewSession, parsed_context: SupersetParsedContext, - findings: List[ValidationFinding], - ) -> tuple[List[ImportedFilter], List[TemplateVariable], List[ExecutionMapping], List[ValidationFinding]]: + findings: list[ValidationFinding], + ) -> tuple[list[ImportedFilter], list[TemplateVariable], list[ExecutionMapping], list[ValidationFinding]]: session_record = cast(Any, session) extractor = SupersetContextExtractor(environment) imported_filters_payload = extractor.recover_imported_filters(parsed_context) @@ -488,8 +480,8 @@ class DatasetReviewOrchestrator: for index, item in enumerate(imported_filters_payload) ] - template_variables: List[TemplateVariable] = [] - execution_mappings: List[ExecutionMapping] = [] + template_variables: list[TemplateVariable] = [] + execution_mappings: list[ExecutionMapping] = [] if session.dataset_id is not None: try: @@ -562,14 +554,14 @@ class DatasetReviewOrchestrator: self, command: StartSessionCommand, session: DatasetReviewSession, - parsed_context: Optional[SupersetParsedContext], - ) -> Optional[str]: + parsed_context: SupersetParsedContext | None, + ) -> str | None: session_record = cast(Any, session) if self.task_manager is None: logger.reason("Dataset review session started without task manager; continuing synchronously", extra={"session_id": session_record.session_id}) return None - task_params: Dict[str, Any] = { + task_params: dict[str, Any] = { "session_id": session_record.session_id, "user_id": command.user.id, "environment_id": session_record.environment_id, diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_commands.py b/backend/src/services/dataset_review/orchestrator_pkg/_commands.py index e9b16c0e..d4790c77 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_commands.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_commands.py @@ -7,8 +7,8 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from src.core.utils.superset_context_extractor import SupersetParsedContext from src.models.auth import User from src.models.dataset_review import ( CompiledPreview, @@ -16,7 +16,6 @@ from src.models.dataset_review import ( DatasetRunContext, ValidationFinding, ) -from src.core.utils.superset_context_extractor import SupersetParsedContext # #region StartSessionCommand [C:2] [TYPE Class] @@ -37,8 +36,8 @@ class StartSessionCommand: @dataclass class StartSessionResult: session: DatasetReviewSession - parsed_context: Optional[SupersetParsedContext] = None - findings: List[ValidationFinding] = field(default_factory=list) + parsed_context: SupersetParsedContext | None = None + findings: list[ValidationFinding] = field(default_factory=list) # #endregion StartSessionResult @@ -50,7 +49,7 @@ class StartSessionResult: class PreparePreviewCommand: user: User session_id: str - expected_version: Optional[int] = None + expected_version: int | None = None # #endregion PreparePreviewCommand @@ -62,7 +61,7 @@ class PreparePreviewCommand: class PreparePreviewResult: session: DatasetReviewSession preview: CompiledPreview - blocked_reasons: List[str] = field(default_factory=list) + blocked_reasons: list[str] = field(default_factory=list) # #endregion PreparePreviewResult @@ -74,7 +73,7 @@ class PreparePreviewResult: class LaunchDatasetCommand: user: User session_id: str - expected_version: Optional[int] = None + expected_version: int | None = None # #endregion LaunchDatasetCommand @@ -86,7 +85,7 @@ class LaunchDatasetCommand: class LaunchDatasetResult: session: DatasetReviewSession run_context: DatasetRunContext - blocked_reasons: List[str] = field(default_factory=list) + blocked_reasons: list[str] = field(default_factory=list) # #endregion LaunchDatasetResult diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py index e3bf5999..2f88fc44 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py @@ -11,30 +11,21 @@ from __future__ import annotations import hashlib import json from datetime import datetime -from typing import Any, Dict, List, Optional, cast +from typing import Any, cast -from src.core.logger import belief_scope, logger +from src.core.logger import logger from src.models.dataset_review import ( ApprovalState, + BusinessSummarySource, CompiledPreview, ConfidenceState, DatasetProfile, DatasetReviewSession, - ExecutionMapping, - FilterConfidenceState, - FilterRecoveryStatus, - FilterSource, FindingArea, FindingSeverity, - ImportedFilter, - MappingMethod, - MappingStatus, PreviewStatus, ResolutionState, - TemplateVariable, ValidationFinding, - VariableKind, - BusinessSummarySource, ) logger = cast(Any, logger) @@ -42,7 +33,7 @@ logger = cast(Any, logger) # #region parse_dataset_selection [C:2] [TYPE Function] # @BRIEF Normalize dataset-selection payload into canonical session references. -def parse_dataset_selection(source_input: str) -> tuple[str, Optional[int]]: +def parse_dataset_selection(source_input: str) -> tuple[str, int | None]: normalized = str(source_input or "").strip() if not normalized: raise ValueError("dataset selection input must be non-empty") @@ -64,7 +55,7 @@ def parse_dataset_selection(source_input: str) -> tuple[str, Optional[int]]: # @BRIEF Create the first profile snapshot so exports and detail views remain usable immediately after intake. def build_initial_profile( session_id: str, - parsed_context: Optional[Any], + parsed_context: Any | None, dataset_ref: str, ) -> DatasetProfile: dataset_name = ( @@ -106,8 +97,8 @@ def build_initial_profile( # @BRIEF Project partial Superset intake recovery into explicit findings without blocking session usability. # @PRE: parsed_context.partial_recovery is true. # @POST: Returns warning-level findings that preserve usable but incomplete state. -def build_partial_recovery_findings(parsed_context: Any) -> List[ValidationFinding]: - findings: List[ValidationFinding] = [] +def build_partial_recovery_findings(parsed_context: Any) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] for unresolved_ref in getattr(parsed_context, "unresolved_references", []): findings.append( ValidationFinding( @@ -149,7 +140,7 @@ def extract_effective_filter_value( # @BRIEF Build effective filters, template params, approvals, and fingerprint for preview and launch gating. # @PRE: Session aggregate includes imported filters, template variables, and current execution mappings. # @POST: Returns deterministic execution snapshot for current session state without mutating persistence. -def build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]: +def build_execution_snapshot(session: DatasetReviewSession) -> dict[str, Any]: session_record = cast(Any, session) filter_lookup = { item.filter_id: item for item in session_record.imported_filters @@ -158,11 +149,11 @@ def build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]: item.variable_id: item for item in session_record.template_variables } - effective_filters: List[Dict[str, Any]] = [] - template_params: Dict[str, Any] = {} - approved_mapping_ids: List[str] = [] - open_warning_refs: List[str] = [] - preview_blockers: List[str] = [] + effective_filters: list[dict[str, Any]] = [] + template_params: dict[str, Any] = {} + approved_mapping_ids: list[str] = [] + open_warning_refs: list[str] = [] + preview_blockers: list[str] = [] mapped_filter_ids: set[str] = set() for mapping in session_record.execution_mappings: @@ -280,9 +271,9 @@ def build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]: # @POST: Returns explicit blocker codes for every unmet launch invariant. def build_launch_blockers( session: DatasetReviewSession, - execution_snapshot: Dict[str, Any], - preview: Optional[CompiledPreview], -) -> List[str]: + execution_snapshot: dict[str, Any], + preview: CompiledPreview | None, +) -> list[str]: session_record = cast(Any, session) blockers = list(execution_snapshot["preview_blockers"]) @@ -316,7 +307,7 @@ def build_launch_blockers( # #region get_latest_preview [C:2] [TYPE Function] # @BRIEF Resolve the current latest preview snapshot for one session aggregate. -def get_latest_preview(session: DatasetReviewSession) -> Optional[CompiledPreview]: +def get_latest_preview(session: DatasetReviewSession) -> CompiledPreview | None: session_record = cast(Any, session) if not session_record.previews: return None @@ -336,7 +327,7 @@ def get_latest_preview(session: DatasetReviewSession) -> Optional[CompiledPrevie # #region compute_preview_fingerprint [C:1] [TYPE Function] # @BRIEF Produce deterministic execution fingerprint for preview truth and staleness checks. -def compute_preview_fingerprint(payload: Dict[str, Any]) -> str: +def compute_preview_fingerprint(payload: dict[str, Any]) -> str: serialized = json.dumps(payload, sort_keys=True, default=str) return hashlib.sha256(serialized.encode("utf-8")).hexdigest() diff --git a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py index 19ad57fe..6c00084b 100644 --- a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py +++ b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py @@ -1,26 +1,26 @@ +from pathlib import Path + import pytest from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import sessionmaker -from pathlib import Path + from src.core.database import _ensure_dataset_review_session_columns -from src.models.mapping import Base, Environment from src.models.auth import User from src.models.dataset_review import ( - DatasetReviewSession, - DatasetProfile, - ValidationFinding, - CompiledPreview, - DatasetRunContext, - ImportedFilter, BusinessSummarySource, + CompiledPreview, ConfidenceState, + DatasetProfile, + DatasetReviewSession, + DatasetRunContext, FindingArea, FindingSeverity, - ReadinessState, - RecommendedAction, + ImportedFilter, SessionCollaborator, SessionCollaboratorRole, + ValidationFinding, ) +from src.models.mapping import Base, Environment from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionRepository, DatasetReviewSessionVersionConflictError, diff --git a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py index a417220b..def152b2 100644 --- a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py +++ b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py @@ -8,24 +8,18 @@ from __future__ import annotations -from datetime import datetime -from typing import Any, List, Optional, cast +from typing import Any, cast from sqlalchemy.orm import Session from src.core.logger import belief_scope, logger from src.models.dataset_review import ( - ClarificationQuestion, - ClarificationSession, CompiledPreview, DatasetProfile, DatasetReviewSession, DatasetRunContext, ExecutionMapping, ImportedFilter, - SemanticFieldEntry, - SessionCollaborator, - SessionEvent, TemplateVariable, ValidationFinding, ) @@ -48,8 +42,8 @@ def save_profile_and_findings( session_id: str, user_id: str, profile: DatasetProfile, - findings: List[ValidationFinding], - expected_version: Optional[int] = None, + findings: list[ValidationFinding], + expected_version: int | None = None, ) -> DatasetReviewSession: with belief_scope("save_profile_and_findings"): session = get_owned_session(session_id, user_id) @@ -71,7 +65,6 @@ def save_profile_and_findings( commit_session_mutation(session, expected_version=expected_version) logger.reflect("Dataset profile and validation findings committed", extra={"session_id": session.session_id, "user_id": user_id, "findings_count": len(findings)}) - from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository return session @@ -91,10 +84,10 @@ def save_recovery_state( load_session_detail_fn, session_id: str, user_id: str, - imported_filters: List[ImportedFilter], - template_variables: List[TemplateVariable], - execution_mappings: List[ExecutionMapping], - expected_version: Optional[int] = None, + imported_filters: list[ImportedFilter], + template_variables: list[TemplateVariable], + execution_mappings: list[ExecutionMapping], + expected_version: int | None = None, ) -> DatasetReviewSession: with belief_scope("save_recovery_state"): session = get_owned_session(session_id, user_id) @@ -138,7 +131,7 @@ def save_preview( session_id: str, user_id: str, preview: CompiledPreview, - expected_version: Optional[int] = None, + expected_version: int | None = None, ) -> CompiledPreview: with belief_scope("save_preview"): session = get_owned_session(session_id, user_id) @@ -173,7 +166,7 @@ def save_run_context( session_id: str, user_id: str, run_context: DatasetRunContext, - expected_version: Optional[int] = None, + expected_version: int | None = None, ) -> DatasetRunContext: with belief_scope("save_run_context"): session = get_owned_session(session_id, user_id) diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index f05c9ce5..e8d72c54 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -15,26 +15,27 @@ # @REJECTED: Keeping all repository operations in one file because it exceeded the fractal limit. from datetime import datetime -from typing import Any, Optional, List, cast +from typing import Any, cast + from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload from sqlalchemy.orm.exc import StaleDataError + +from src.core.logger import belief_scope, logger from src.models.dataset_review import ( ClarificationQuestion, ClarificationSession, - DatasetReviewSession, - DatasetProfile, - ValidationFinding, CompiledPreview, + DatasetProfile, + DatasetReviewSession, DatasetRunContext, ExecutionMapping, ImportedFilter, SemanticFieldEntry, SessionCollaborator, - SessionEvent, TemplateVariable, + ValidationFinding, ) -from src.core.logger import belief_scope, logger from src.services.dataset_review.event_logger import SessionEventLogger logger = cast(Any, logger) @@ -128,7 +129,7 @@ class DatasetReviewSessionRepository: def bump_session_version(self, session: DatasetReviewSession) -> int: with belief_scope("DatasetReviewSessionRepository.bump_session_version"): next_version = int(getattr(session, "version", 0) or 0) + 1 - setattr(session, "version", next_version) + session.version = next_version session.last_activity_at = datetime.utcnow() logger.reflect("Prepared incremented session version", extra={"session_id": session.session_id, "version": next_version}) return next_version @@ -139,7 +140,7 @@ class DatasetReviewSessionRepository: # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts. # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised. def commit_session_mutation( - self, session: DatasetReviewSession, *, refresh_targets: Optional[List[Any]] = None, expected_version: Optional[int] = None, + self, session: DatasetReviewSession, *, refresh_targets: list[Any] | None = None, expected_version: int | None = None, ) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.commit_session_mutation"): observed_version = int(expected_version if expected_version is not None else getattr(session, "version", 0) or 0) @@ -164,7 +165,7 @@ class DatasetReviewSessionRepository: # region load_detail [TYPE Function] # @PURPOSE: Return the full session aggregate for API and frontend resume flows. # @POST: Returns SessionDetail with all fields populated or None. - def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]: + def load_session_detail(self, session_id: str, user_id: str) -> DatasetReviewSession | None: with belief_scope("DatasetReviewSessionRepository.load_session_detail"): logger.reason("Loading dataset review session detail", extra={"session_id": session_id, "user_id": user_id}) session = ( @@ -198,7 +199,7 @@ class DatasetReviewSessionRepository: # @PURPOSE: Persist profile state and replace validation findings for an owned session. # @POST: stored profile matches the current session and findings are replaced. def save_profile_and_findings( - self, session_id: str, user_id: str, profile: DatasetProfile, findings: List[ValidationFinding], expected_version: Optional[int] = None, + self, session_id: str, user_id: str, profile: DatasetProfile, findings: list[ValidationFinding], expected_version: int | None = None, ) -> DatasetReviewSession: from src.services.dataset_review.repositories.repository_pkg._mutations import save_profile_and_findings as _save return _save( @@ -211,9 +212,9 @@ class DatasetReviewSessionRepository: # region save_recovery_state [TYPE Function] # @PURPOSE: Persist imported filters, template variables, and initial execution mappings. def save_recovery_state( - self, session_id: str, user_id: str, imported_filters: List[ImportedFilter], - template_variables: List[TemplateVariable], execution_mappings: List[ExecutionMapping], - expected_version: Optional[int] = None, + self, session_id: str, user_id: str, imported_filters: list[ImportedFilter], + template_variables: list[TemplateVariable], execution_mappings: list[ExecutionMapping], + expected_version: int | None = None, ) -> DatasetReviewSession: from src.services.dataset_review.repositories.repository_pkg._mutations import save_recovery_state as _save return _save( @@ -227,7 +228,7 @@ class DatasetReviewSessionRepository: # region save_preview [TYPE Function] # @PURPOSE: Persist a preview snapshot and mark prior session previews stale. def save_preview( - self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: Optional[int] = None, + self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: int | None = None, ) -> CompiledPreview: from src.services.dataset_review.repositories.repository_pkg._mutations import save_preview as _save return _save( @@ -240,7 +241,7 @@ class DatasetReviewSessionRepository: # region save_run_context [TYPE Function] # @PURPOSE: Persist an immutable launch audit snapshot for an owned session. def save_run_context( - self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: Optional[int] = None, + self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: int | None = None, ) -> DatasetRunContext: from src.services.dataset_review.repositories.repository_pkg._mutations import save_run_context as _save return _save( @@ -252,7 +253,7 @@ class DatasetReviewSessionRepository: # region list_user_sess [TYPE Function] # @PURPOSE: List review sessions owned by a specific user ordered by most recent update. - def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]: + def list_sessions_for_user(self, user_id: str) -> list[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.list_sessions_for_user"): logger.reason("Listing dataset review sessions for owner scope", extra={"user_id": user_id}) sessions = ( diff --git a/backend/src/services/dataset_review/semantic_resolver.py b/backend/src/services/dataset_review/semantic_resolver.py index a490fd41..0a697bf0 100644 --- a/backend/src/services/dataset_review/semantic_resolver.py +++ b/backend/src/services/dataset_review/semantic_resolver.py @@ -12,10 +12,12 @@ from __future__ import annotations +from collections.abc import Iterable, Mapping + # #region imports [TYPE Block] from dataclasses import dataclass, field from difflib import SequenceMatcher -from typing import Any, Dict, Iterable, List, Mapping, Optional +from typing import Any from src.core.logger import belief_scope, logger from src.models.dataset_review import ( @@ -24,6 +26,7 @@ from src.models.dataset_review import ( FieldProvenance, SemanticSource, ) + # #endregion imports @@ -32,8 +35,8 @@ from src.models.dataset_review import ( @dataclass class DictionaryResolutionResult: source_ref: str - resolved_fields: List[Dict[str, Any]] = field(default_factory=list) - unresolved_fields: List[str] = field(default_factory=list) + resolved_fields: list[dict[str, Any]] = field(default_factory=list) + unresolved_fields: list[str] = field(default_factory=list) partial_recovery: bool = False # #endregion DictionaryResolutionResult @@ -92,8 +95,8 @@ class SemanticSourceResolver: if row.get("field_key") } - resolved_fields: List[Dict[str, Any]] = [] - unresolved_fields: List[str] = [] + resolved_fields: list[dict[str, Any]] = [] + unresolved_fields: list[str] = [] for raw_field in fields: field_name = str(raw_field.get("field_name") or "").strip() @@ -121,7 +124,7 @@ class SemanticSourceResolver: continue exact_match = row_index.get(self._normalize_key(field_name)) - candidates: List[Dict[str, Any]] = [] + candidates: list[dict[str, Any]] = [] if exact_match is not None: logger.reason( @@ -222,7 +225,7 @@ class SemanticSourceResolver: # region rank_candidates [TYPE Function] # @PURPOSE: Apply confidence ordering and determine best candidate per field. # @RELATION: [DEPENDS_ON] ->[SemanticCandidate] - def rank_candidates(self, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def rank_candidates(self, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: ranked = sorted( candidates, key=lambda candidate: ( @@ -238,13 +241,13 @@ class SemanticSourceResolver: # region detect_conflicts [TYPE Function] # @PURPOSE: Mark competing candidate sets that require explicit user review. - def detect_conflicts(self, candidates: List[Dict[str, Any]]) -> bool: + def detect_conflicts(self, candidates: list[dict[str, Any]]) -> bool: return len(candidates) > 1 # endregion detect_conflicts # region apply_field_decision [TYPE Function] # @PURPOSE: Accept, reject, or manually override a field-level semantic value. - def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> Dict[str, Any]: + def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> dict[str, Any]: merged = dict(field_state) merged.update(decision) return merged @@ -262,7 +265,7 @@ class SemanticSourceResolver: self, source: SemanticSource, fields: Iterable[Any], - ) -> Dict[str, int]: + ) -> dict[str, int]: with belief_scope("SemanticSourceResolver.propagate_source_version_update"): source_id = str(source.source_id or "").strip() source_version = str(source.source_version or "").strip() @@ -308,7 +311,7 @@ class SemanticSourceResolver: # region _normalize_dictionary_row [TYPE Function] # @PURPOSE: Normalize one dictionary row into a consistent lookup structure. - def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> Dict[str, Any]: + def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> dict[str, Any]: field_name = ( row.get("field_name") or row.get("column_name") @@ -327,9 +330,9 @@ class SemanticSourceResolver: # region _find_fuzzy_matches [TYPE Function] # @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable. - def _find_fuzzy_matches(self, field_name: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def _find_fuzzy_matches(self, field_name: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: normalized_target = self._normalize_key(field_name) - fuzzy_matches: List[Dict[str, Any]] = [] + fuzzy_matches: list[dict[str, Any]] = [] for row in rows: candidate_key = str(row.get("field_key") or "") if not candidate_key: @@ -350,7 +353,7 @@ class SemanticSourceResolver: match_type: CandidateMatchType, confidence_score: float, row: Mapping[str, Any], - ) -> Dict[str, Any]: + ) -> dict[str, Any]: return { "candidate_rank": rank, "match_type": match_type.value, @@ -364,7 +367,7 @@ class SemanticSourceResolver: # region _match_priority [TYPE Function] # @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention. - def _match_priority(self, match_type: Optional[str]) -> int: + def _match_priority(self, match_type: str | None) -> int: priority = { CandidateMatchType.EXACT.value: 0, CandidateMatchType.REFERENCE.value: 1, @@ -381,4 +384,4 @@ class SemanticSourceResolver: # endregion _normalize_key # #endregion SemanticSourceResolver -# #endregion SemanticSourceResolver \ No newline at end of file +# #endregion SemanticSourceResolver diff --git a/backend/src/services/git/__init__.py b/backend/src/services/git/__init__.py index 8f41a8ad..cb10d5f2 100644 --- a/backend/src/services/git/__init__.py +++ b/backend/src/services/git/__init__.py @@ -19,12 +19,12 @@ from ._base import GitServiceBase from ._branch import GitServiceBranchMixin -from ._sync import GitServiceSyncMixin -from ._status import GitServiceStatusMixin -from ._merge import GitServiceMergeMixin -from ._url import GitServiceUrlMixin from ._gitea import GitServiceGiteaMixin +from ._merge import GitServiceMergeMixin from ._remote_providers import GitServiceGithubMixin, GitServiceGitlabMixin +from ._status import GitServiceStatusMixin +from ._sync import GitServiceSyncMixin +from ._url import GitServiceUrlMixin __all__ = ["GitService"] diff --git a/backend/src/services/git/_base.py b/backend/src/services/git/_base.py index 03bbdc3e..2f085388 100644 --- a/backend/src/services/git/_base.py +++ b/backend/src/services/git/_base.py @@ -10,14 +10,15 @@ import os import re import shutil from pathlib import Path -from typing import Any, Dict, List, Optional + +from fastapi import HTTPException from git import Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError -from fastapi import HTTPException -from src.core.logger import logger, belief_scope -from src.models.git import GitRepository -from src.models.config import AppConfigRecord + from src.core.database import SessionLocal +from src.core.logger import belief_scope, logger +from src.models.config import AppConfigRecord +from src.models.git import GitRepository # #region GitServiceBase [C:3] [TYPE Class] @@ -94,7 +95,7 @@ class GitServiceBase: # @PRE: repo_key can be None/empty. # @POST: Returns normalized non-empty key. # @RETURN: str - def _normalize_repo_key(self, repo_key: Optional[str]) -> str: + def _normalize_repo_key(self, repo_key: str | None) -> str: raw_key = str(repo_key or "").strip().lower() normalized = re.sub(r"[^a-z0-9._-]+", "-", raw_key).strip("._-") return normalized or "dashboard" @@ -154,7 +155,7 @@ class GitServiceBase: # @PRE: dashboard_id is an integer. # @POST: Returns DB-local_path when present, otherwise base_path/. # @RETURN: str - def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str: + def _get_repo_path(self, dashboard_id: int, repo_key: str | None = None) -> str: with belief_scope("GitService._get_repo_path"): if dashboard_id is None: raise ValueError("dashboard_id cannot be None") @@ -199,7 +200,7 @@ class GitServiceBase: # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided. # @POST: Repository is cloned or opened at the local path. # @RETURN: Repo - def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo: + def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: str | None = None) -> Repo: with belief_scope("GitService.init_repo"): self._ensure_base_path_exists() repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id)) @@ -269,7 +270,7 @@ class GitServiceBase: except Exception as e: session.rollback() logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to delete repository: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to delete repository: {e!s}") finally: session.close() # endregion delete_repo @@ -296,7 +297,7 @@ class GitServiceBase: # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations. # @PRE: dashboard_id repository exists; git_username/git_email may be empty. # @POST: Repository config has user.name and user.email when both identity values are provided. - def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None: + def configure_identity(self, dashboard_id: int, git_username: str | None, git_email: str | None) -> None: with belief_scope("GitService.configure_identity"): normalized_username = str(git_username or "").strip() normalized_email = str(git_email or "").strip() @@ -310,7 +311,7 @@ class GitServiceBase: logger.reason(f"Applied repository-local git identity for dashboard {dashboard_id}", extra={"src": "configure_identity"}) except Exception as e: logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}") - raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {e!s}") # endregion configure_identity # #endregion GitServiceBase # #endregion GitServiceBase diff --git a/backend/src/services/git/_branch.py b/backend/src/services/git/_branch.py index fe5994be..e12747f2 100644 --- a/backend/src/services/git/_branch.py +++ b/backend/src/services/git/_branch.py @@ -4,12 +4,12 @@ # @RELATION USED_BY -> [GitService] import os -from typing import Any, Dict, List, Optional from datetime import datetime -from git import Repo -from git.exc import GitCommandError + from fastapi import HTTPException -from src.core.logger import logger, belief_scope +from git import Repo + +from src.core.logger import belief_scope, logger # #region GitServiceBranchMixin [C:3] [TYPE Class] @@ -77,7 +77,7 @@ class GitServiceBranchMixin: logger.error(f"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}") raise HTTPException( status_code=500, - detail=f"Failed to create default branch '{branch_name}' on remote: {str(e)}", + detail=f"Failed to create default branch '{branch_name}' on remote: {e!s}", ) try: repo.git.checkout("dev") @@ -91,7 +91,7 @@ class GitServiceBranchMixin: # @PRE: Repository for dashboard_id exists. # @POST: Returns a list of branch metadata dictionaries. # @RETURN: List[dict] - def list_branches(self, dashboard_id: int) -> List[dict]: + def list_branches(self, dashboard_id: int) -> list[dict]: with belief_scope("GitService.list_branches"): repo = self.get_repo(dashboard_id) logger.reason(f"Listing branches for {dashboard_id}. Refs: {repo.refs}", extra={"src": "list_branches"}) @@ -174,7 +174,7 @@ class GitServiceBranchMixin: # @PARAM: files (List[str]) - Optional list of specific files to stage. # @PRE: Repository exists and has changes (dirty) or files are specified. # @POST: Changes are staged and a new commit is created. - def commit_changes(self, dashboard_id: int, message: str, files: List[str] = None): + def commit_changes(self, dashboard_id: int, message: str, files: list[str] = None): with belief_scope("GitService.commit_changes"): repo = self.get_repo(dashboard_id) if not repo.is_dirty(untracked_files=True) and not files: diff --git a/backend/src/services/git/_gitea.py b/backend/src/services/git/_gitea.py index 74b1c0ba..b90ba9b1 100644 --- a/backend/src/services/git/_gitea.py +++ b/backend/src/services/git/_gitea.py @@ -3,11 +3,13 @@ # @BRIEF Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation. # @RELATION USED_BY -> [GitService] -import httpx -from typing import Any, Dict, List, Optional +from typing import Any from urllib.parse import quote + +import httpx from fastapi import HTTPException -from src.core.logger import logger, belief_scope + +from src.core.logger import belief_scope, logger from src.models.git import GitProvider @@ -61,7 +63,7 @@ class GitServiceGiteaMixin: # @PRE: pat is provided. # @POST: Returns headers with token auth. # @RETURN: Dict[str, str] - def _gitea_headers(self, pat: str) -> Dict[str, str]: + def _gitea_headers(self, pat: str) -> dict[str, str]: token = (pat or "").strip() if not token: raise HTTPException(status_code=400, detail="Git PAT is required for Gitea operations") @@ -79,7 +81,7 @@ class GitServiceGiteaMixin: # @RETURN: Any async def _gitea_request( self, method: str, server_url: str, pat: str, endpoint: str, - payload: Optional[Dict[str, Any]] = None, + payload: dict[str, Any] | None = None, ) -> Any: base_url = self._normalize_git_server_url(server_url) url = f"{base_url}/api/v1{endpoint}" @@ -89,7 +91,7 @@ class GitServiceGiteaMixin: response = await client.request(method=method, url=url, headers=headers, json=payload) except Exception as e: logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}") - raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {str(e)}") + raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {e!s}") if response.status_code >= 400: detail = response.text try: @@ -122,7 +124,7 @@ class GitServiceGiteaMixin: # @PRE: server_url and pat are valid. # @POST: Returns repository list from Gitea. # @RETURN: List[dict] - async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]: + async def list_gitea_repositories(self, server_url: str, pat: str) -> list[dict]: payload = await self._gitea_request("GET", server_url, pat, "/user/repos?limit=100&page=1") if not isinstance(payload, list): return [] @@ -136,9 +138,9 @@ class GitServiceGiteaMixin: # @RETURN: dict async def create_gitea_repository( self, server_url: str, pat: str, name: str, private: bool = True, - description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", - ) -> Dict[str, Any]: - payload: Dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)} + description: str | None = None, auto_init: bool = True, default_branch: str | None = "main", + ) -> dict[str, Any]: + payload: dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)} if description: payload["description"] = description if default_branch: @@ -184,7 +186,7 @@ class GitServiceGiteaMixin: # @RETURN: Optional[str] async def _build_gitea_pr_404_detail( self, server_url: str, pat: str, owner: str, repo: str, from_branch: str, to_branch: str, - ) -> Optional[str]: + ) -> str | None: source_exists = await self._gitea_branch_exists( server_url=server_url, pat=pat, owner=owner, repo=repo, branch=from_branch, ) @@ -205,8 +207,8 @@ class GitServiceGiteaMixin: # @RETURN: Dict[str, Any] async def create_gitea_pull_request( self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, - title: str, description: Optional[str] = None, - ) -> Dict[str, Any]: + title: str, description: str | None = None, + ) -> dict[str, Any]: identity = self._parse_remote_repo_identity(remote_url) req_payload = {"title": title, "head": from_branch, "base": to_branch, "body": description or ""} endpoint = f"/repos/{identity['owner']}/{identity['repo']}/pulls" diff --git a/backend/src/services/git/_merge.py b/backend/src/services/git/_merge.py index 85b47bb9..00b4cef6 100644 --- a/backend/src/services/git/_merge.py +++ b/backend/src/services/git/_merge.py @@ -5,12 +5,14 @@ import os from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any + +from fastapi import HTTPException from git import Repo from git.exc import GitCommandError from git.objects.blob import Blob -from fastapi import HTTPException -from src.core.logger import logger, belief_scope + +from src.core.logger import belief_scope # #region GitServiceMergeMixin [C:3] [TYPE Class] @@ -30,7 +32,7 @@ class GitServiceMergeMixin: # region _get_unmerged_file_paths [TYPE Function] # @PURPOSE: List files with merge conflicts. - def _get_unmerged_file_paths(self, repo: Repo) -> List[str]: + def _get_unmerged_file_paths(self, repo: Repo) -> list[str]: with belief_scope("GitService._get_unmerged_file_paths"): try: return sorted(list(repo.index.unmerged_blobs().keys())) @@ -40,7 +42,7 @@ class GitServiceMergeMixin: # region _build_unfinished_merge_payload [TYPE Function] # @PURPOSE: Build payload for unfinished merge state. - def _build_unfinished_merge_payload(self, repo: Repo) -> Dict[str, Any]: + def _build_unfinished_merge_payload(self, repo: Repo) -> dict[str, Any]: with belief_scope("GitService._build_unfinished_merge_payload"): merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") merge_head_value = "" @@ -87,7 +89,7 @@ class GitServiceMergeMixin: # region get_merge_status [TYPE Function] # @PURPOSE: Get current merge status for a dashboard repository. - def get_merge_status(self, dashboard_id: int) -> Dict[str, Any]: + def get_merge_status(self, dashboard_id: int) -> dict[str, Any]: with belief_scope("GitService.get_merge_status"): repo = self.get_repo(dashboard_id) merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") @@ -120,7 +122,7 @@ class GitServiceMergeMixin: # region get_merge_conflicts [TYPE Function] # @PURPOSE: List all files with conflicts and their contents. - def get_merge_conflicts(self, dashboard_id: int) -> List[Dict[str, Any]]: + def get_merge_conflicts(self, dashboard_id: int) -> list[dict[str, Any]]: with belief_scope("GitService.get_merge_conflicts"): repo = self.get_repo(dashboard_id) conflicts = [] @@ -143,10 +145,10 @@ class GitServiceMergeMixin: # region resolve_merge_conflicts [TYPE Function] # @PURPOSE: Resolve conflicts using specified strategy. - def resolve_merge_conflicts(self, dashboard_id: int, resolutions: List[Dict[str, Any]]) -> List[str]: + def resolve_merge_conflicts(self, dashboard_id: int, resolutions: list[dict[str, Any]]) -> list[str]: with belief_scope("GitService.resolve_merge_conflicts"): repo = self.get_repo(dashboard_id) - resolved_files: List[str] = [] + resolved_files: list[str] = [] repo_root = os.path.abspath(str(repo.working_tree_dir or "")) if not repo_root: raise HTTPException(status_code=500, detail="Repository working tree directory is unavailable") @@ -176,7 +178,7 @@ class GitServiceMergeMixin: # region abort_merge [TYPE Function] # @PURPOSE: Abort ongoing merge. - def abort_merge(self, dashboard_id: int) -> Dict[str, Any]: + def abort_merge(self, dashboard_id: int) -> dict[str, Any]: with belief_scope("GitService.abort_merge"): repo = self.get_repo(dashboard_id) try: @@ -192,7 +194,7 @@ class GitServiceMergeMixin: # region continue_merge [TYPE Function] # @PURPOSE: Finalize merge after conflict resolution. - def continue_merge(self, dashboard_id: int, message: Optional[str] = None) -> Dict[str, Any]: + def continue_merge(self, dashboard_id: int, message: str | None = None) -> dict[str, Any]: with belief_scope("GitService.continue_merge"): repo = self.get_repo(dashboard_id) unmerged_files = self._get_unmerged_file_paths(repo) @@ -230,7 +232,7 @@ class GitServiceMergeMixin: # @PRE: Repository exists and both branches are valid. # @POST: Target branch contains merged changes from source branch. # @RETURN: Dict[str, Any] - def promote_direct_merge(self, dashboard_id: int, from_branch: str, to_branch: str) -> Dict[str, Any]: + def promote_direct_merge(self, dashboard_id: int, from_branch: str, to_branch: str) -> dict[str, Any]: with belief_scope("GitService.promote_direct_merge"): if not from_branch or not to_branch: raise HTTPException(status_code=400, detail="from_branch and to_branch are required") diff --git a/backend/src/services/git/_remote_providers.py b/backend/src/services/git/_remote_providers.py index 047255e7..506322ef 100644 --- a/backend/src/services/git/_remote_providers.py +++ b/backend/src/services/git/_remote_providers.py @@ -3,11 +3,11 @@ # @BRIEF GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation. # @RELATION USED_BY -> [GitService] -import httpx -from typing import Any, Dict, Optional +from typing import Any from urllib.parse import quote + +import httpx from fastapi import HTTPException -from src.core.logger import logger, belief_scope # #region GitServiceGithubMixin [C:3] [TYPE Class] @@ -20,8 +20,8 @@ class GitServiceGithubMixin: # @RETURN: dict async def create_github_repository( self, server_url: str, pat: str, name: str, private: bool = True, - description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", - ) -> Dict[str, Any]: + description: str | None = None, auto_init: bool = True, default_branch: str | None = "main", + ) -> dict[str, Any]: base_url = self._normalize_git_server_url(server_url) if "github.com" in base_url: api_url = "https://api.github.com/user/repos" @@ -32,7 +32,7 @@ class GitServiceGithubMixin: "Content-Type": "application/json", "Accept": "application/vnd.github+json", } - payload: Dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)} + payload: dict[str, Any] = {"name": name, "private": bool(private), "auto_init": bool(auto_init)} if description: payload["description"] = description if default_branch: @@ -41,7 +41,7 @@ class GitServiceGithubMixin: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.post(api_url, headers=headers, json=payload) except Exception as e: - raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}") + raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {e!s}") if response.status_code >= 400: detail = response.text try: @@ -60,8 +60,8 @@ class GitServiceGithubMixin: # @RETURN: Dict[str, Any] async def create_github_pull_request( self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, - title: str, description: Optional[str] = None, draft: bool = False, - ) -> Dict[str, Any]: + title: str, description: str | None = None, draft: bool = False, + ) -> dict[str, Any]: identity = self._parse_remote_repo_identity(remote_url) base_url = self._normalize_git_server_url(server_url) if "github.com" in base_url: @@ -81,7 +81,7 @@ class GitServiceGithubMixin: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.post(api_url, headers=headers, json=payload) except Exception as e: - raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {str(e)}") + raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {e!s}") if response.status_code >= 400: detail = response.text try: @@ -104,12 +104,12 @@ class GitServiceGitlabMixin: # @RETURN: dict async def create_gitlab_repository( self, server_url: str, pat: str, name: str, private: bool = True, - description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", - ) -> Dict[str, Any]: + description: str | None = None, auto_init: bool = True, default_branch: str | None = "main", + ) -> dict[str, Any]: base_url = self._normalize_git_server_url(server_url) api_url = f"{base_url}/api/v4/projects" headers = {"PRIVATE-TOKEN": pat.strip(), "Content-Type": "application/json", "Accept": "application/json"} - payload: Dict[str, Any] = { + payload: dict[str, Any] = { "name": name, "visibility": "private" if private else "public", "initialize_with_readme": bool(auto_init), } @@ -121,7 +121,7 @@ class GitServiceGitlabMixin: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.post(api_url, headers=headers, json=payload) except Exception as e: - raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}") + raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {e!s}") if response.status_code >= 400: detail = response.text try: @@ -150,8 +150,8 @@ class GitServiceGitlabMixin: # @RETURN: Dict[str, Any] async def create_gitlab_merge_request( self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, - title: str, description: Optional[str] = None, remove_source_branch: bool = False, - ) -> Dict[str, Any]: + title: str, description: str | None = None, remove_source_branch: bool = False, + ) -> dict[str, Any]: identity = self._parse_remote_repo_identity(remote_url) base_url = self._normalize_git_server_url(server_url) project_id = quote(identity["full_name"], safe="") @@ -165,7 +165,7 @@ class GitServiceGitlabMixin: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.post(api_url, headers=headers, json=payload) except Exception as e: - raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {str(e)}") + raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {e!s}") if response.status_code >= 400: detail = response.text try: diff --git a/backend/src/services/git/_status.py b/backend/src/services/git/_status.py index d8a4e499..d76c91ef 100644 --- a/backend/src/services/git/_status.py +++ b/backend/src/services/git/_status.py @@ -3,10 +3,9 @@ # @BRIEF Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues. # @RELATION USED_BY -> [GitService] -from typing import Any, Dict, List from datetime import datetime -from git import Repo -from src.core.logger import logger, belief_scope + +from src.core.logger import belief_scope, logger # #region GitServiceStatusMixin [C:3] [TYPE Class] @@ -135,7 +134,7 @@ class GitServiceStatusMixin: # @PRE: Repository for dashboard_id exists. # @POST: Returns a list of dictionaries for each commit in history. # @RETURN: List[dict] - def get_commit_history(self, dashboard_id: int, limit: int = 50) -> List[dict]: + def get_commit_history(self, dashboard_id: int, limit: int = 50) -> list[dict]: with belief_scope("GitService.get_commit_history"): repo = self.get_repo(dashboard_id) commits = [] diff --git a/backend/src/services/git/_sync.py b/backend/src/services/git/_sync.py index 635b10cf..546c7923 100644 --- a/backend/src/services/git/_sync.py +++ b/backend/src/services/git/_sync.py @@ -5,12 +5,12 @@ # @RELATION DEPENDS_ON -> [GitServiceUrlMixin] import os -from typing import Optional -from git import Repo -from git.exc import GitCommandError + from fastapi import HTTPException +from git.exc import GitCommandError + from src.core.database import SessionLocal -from src.core.logger import logger, belief_scope +from src.core.logger import belief_scope, logger from src.models.git import GitRepository, GitServerConfig @@ -107,7 +107,7 @@ class GitServiceSyncMixin: raise HTTPException(status_code=500, detail=f"Git push failed: {details}") except Exception as e: logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") - raise HTTPException(status_code=500, detail=f"Git push failed: {str(e)}") + raise HTTPException(status_code=500, detail=f"Git push failed: {e!s}") # endregion push_changes # region pull_changes [TYPE Function] @@ -167,7 +167,7 @@ class GitServiceSyncMixin: raise except Exception as e: logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") - raise HTTPException(status_code=500, detail=f"Git pull failed: {str(e)}") + raise HTTPException(status_code=500, detail=f"Git pull failed: {e!s}") # endregion pull_changes # #endregion GitServiceSyncMixin # #endregion GitServiceSyncMixin diff --git a/backend/src/services/git/_url.py b/backend/src/services/git/_url.py index c9ba566c..686fa54e 100644 --- a/backend/src/services/git/_url.py +++ b/backend/src/services/git/_url.py @@ -5,13 +5,14 @@ # @RELATION USED_BY -> [GitServiceGiteaMixin] # @RELATION USED_BY -> [GitServiceRemoteMixin] -import os -from typing import Any, Dict, List, Optional from urllib.parse import quote, urlparse + from fastapi import HTTPException + from src.core.database import SessionLocal -from src.core.logger import logger, belief_scope -from src.models.git import GitRepository, GitServerConfig +from src.core.logger import logger +from src.models.git import GitRepository + # #region GitServiceUrlMixin [C:3] [TYPE Class] # @BRIEF URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL. @@ -21,7 +22,7 @@ class GitServiceUrlMixin: # @PRE: url_value may be empty. # @POST: Returns lowercase host token or None. # @RETURN: Optional[str] - def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]: + def _extract_http_host(self, url_value: str | None) -> str | None: normalized = str(url_value or "").strip() if not normalized: return None @@ -65,7 +66,7 @@ class GitServiceUrlMixin: # @PRE: source_url and config_url are HTTP(S) URLs. # @POST: Returns source URL with updated host (credentials preserved) or None. # @RETURN: Optional[str] - def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]: + def _replace_host_in_url(self, source_url: str | None, config_url: str | None) -> str | None: source = str(source_url or "").strip() config = str(config_url or "").strip() if not source or not config: @@ -101,10 +102,10 @@ class GitServiceUrlMixin: self, dashboard_id: int, origin, - config_url: Optional[str], - current_origin_url: Optional[str], - binding_remote_url: Optional[str], - ) -> Optional[str]: + config_url: str | None, + current_origin_url: str | None, + binding_remote_url: str | None, + ) -> str | None: config_host = self._extract_http_host(config_url) source_origin_url = str(current_origin_url or "").strip() or str(binding_remote_url or "").strip() origin_host = self._extract_http_host(source_origin_url) @@ -153,7 +154,7 @@ class GitServiceUrlMixin: # @PRE: remote_url is a valid git URL. # @POST: Returns owner/repo tokens. # @RETURN: Dict[str, str] - def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]: + def _parse_remote_repo_identity(self, remote_url: str) -> dict[str, str]: normalized = str(remote_url or "").strip() if not normalized: raise HTTPException(status_code=400, detail="Repository remote_url is empty") @@ -179,7 +180,7 @@ class GitServiceUrlMixin: # @PRE: remote_url may be any git URL. # @POST: Returns normalized http(s) base URL or None when derivation is impossible. # @RETURN: Optional[str] - def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]: + def _derive_server_url_from_remote(self, remote_url: str) -> str | None: normalized = str(remote_url or "").strip() if not normalized or normalized.startswith("git@"): return None diff --git a/backend/src/services/health_service.py b/backend/src/services/health_service.py index cb5f80fd..8ba29695 100644 --- a/backend/src/services/health_service.py +++ b/backend/src/services/health_service.py @@ -6,21 +6,23 @@ # @RELATION DEPENDS_ON -> [TaskCleanupService] # @RELATION DEPENDS_ON -> [TaskManager] -from typing import List, Dict, Any, Optional, Tuple, cast -import time -from sqlalchemy.orm import Session -from sqlalchemy import func import os -from ..models.llm import ValidationRecord -from ..schemas.health import DashboardHealthItem, HealthSummaryResponse +import time +from typing import Any, cast + +from sqlalchemy import func +from sqlalchemy.orm import Session + from ..core.logger import logger from ..core.superset_client import SupersetClient -from ..core.task_manager.cleanup import TaskCleanupService from ..core.task_manager import TaskManager +from ..core.task_manager.cleanup import TaskCleanupService +from ..models.llm import ValidationRecord +from ..schemas.health import DashboardHealthItem, HealthSummaryResponse -def _empty_dashboard_meta() -> Dict[str, Optional[str]]: - return cast(Dict[str, Optional[str]], {"slug": None, "title": None}) +def _empty_dashboard_meta() -> dict[str, str | None]: + return cast(dict[str, str | None], {"slug": None, "title": None}) # #region HealthService [C:4] [TYPE Class] @@ -36,8 +38,8 @@ def _empty_dashboard_meta() -> Dict[str, Optional[str]]: # @RELATION DEPENDS_ON -> [TaskCleanupService] # @RELATION DEPENDS_ON -> [TaskManager] class HealthService: - _dashboard_summary_cache: Dict[ - str, Tuple[float, Dict[str, Dict[str, Optional[str]]]] + _dashboard_summary_cache: dict[ + str, tuple[float, dict[str, dict[str, str | None]]] ] = {} _dashboard_summary_cache_ttl_seconds = 60.0 @@ -55,7 +57,7 @@ class HealthService: def __init__(self, db: Session, config_manager=None): self.db = db self.config_manager = config_manager - self._dashboard_meta_cache: Dict[Tuple[str, str], Dict[str, Optional[str]]] = {} + self._dashboard_meta_cache: dict[tuple[str, str], dict[str, str | None]] = {} # endregion HealthService_init @@ -68,11 +70,11 @@ class HealthService: # @RELATION: [DEPENDS_ON] ->[ValidationRecord] # @RELATION: [DEPENDS_ON] ->[ConfigManager] # @RELATION: [DEPENDS_ON] ->[SupersetClient] - def _prime_dashboard_meta_cache(self, records: List[ValidationRecord]) -> None: + def _prime_dashboard_meta_cache(self, records: list[ValidationRecord]) -> None: if not self.config_manager or not records: return - numeric_ids_by_env: Dict[str, set[str]] = {} + numeric_ids_by_env: dict[str, set[str]] = {} for record in records: environment_id = str(record.environment_id or "").strip() dashboard_id = str(record.dashboard_id or "").strip() @@ -105,14 +107,14 @@ class HealthService: cached_meta = self.__class__._dashboard_summary_cache.get( environment_id ) - dashboard_meta_map: Dict[str, Dict[str, Optional[str]]] + dashboard_meta_map: dict[str, dict[str, str | None]] if ( cached_meta is not None and (time.monotonic() - cached_meta[0]) < self.__class__._dashboard_summary_cache_ttl_seconds ): cached_meta_data = cast( - Tuple[float, Dict[str, Dict[str, Optional[str]]]], + tuple[float, dict[str, dict[str, str | None]]], cached_meta, ) dashboard_meta_map = cached_meta_data[1] @@ -156,8 +158,8 @@ class HealthService: # @POST: Returns dict with `slug` and `title` keys, using cache when possible. # @SIDE_EFFECT: Writes default cache entries for unresolved numeric dashboard ids. def _resolve_dashboard_meta( - self, dashboard_id: str, environment_id: Optional[str] - ) -> Dict[str, Optional[str]]: + self, dashboard_id: str, environment_id: str | None + ) -> dict[str, str | None]: normalized_dashboard_id = str(dashboard_id or "").strip() normalized_environment_id = str(environment_id or "").strip() if not normalized_dashboard_id: @@ -293,7 +295,7 @@ class HealthService: # @RELATION: [DEPENDS_ON] ->[TaskManager] # @RELATION: [DEPENDS_ON] ->[TaskCleanupService] def delete_validation_report( - self, record_id: str, task_manager: Optional[TaskManager] = None + self, record_id: str, task_manager: TaskManager | None = None ) -> bool: record = ( self.db.query(ValidationRecord) diff --git a/backend/src/services/llm_prompt_templates.py b/backend/src/services/llm_prompt_templates.py index 7299e85e..1f59ecbe 100644 --- a/backend/src/services/llm_prompt_templates.py +++ b/backend/src/services/llm_prompt_templates.py @@ -7,12 +7,11 @@ from __future__ import annotations from copy import deepcopy -from typing import Dict, Any, Optional - +from typing import Any # #region DEFAULT_LLM_PROMPTS [C:2] [TYPE Constant] # @BRIEF Default prompt templates used by documentation, dashboard validation, and git commit generation. -DEFAULT_LLM_PROMPTS: Dict[str, str] = { +DEFAULT_LLM_PROMPTS: dict[str, str] = { "dashboard_validation_prompt": ( "Analyze the attached dashboard screenshot and the following execution logs for health and visual issues.\n\n" "Logs:\n" @@ -60,7 +59,7 @@ DEFAULT_LLM_PROMPTS: Dict[str, str] = { # #region DEFAULT_LLM_PROVIDER_BINDINGS [C:2] [TYPE Constant] # @BRIEF Default provider binding per task domain. -DEFAULT_LLM_PROVIDER_BINDINGS: Dict[str, str] = { +DEFAULT_LLM_PROVIDER_BINDINGS: dict[str, str] = { "dashboard_validation": "", "documentation": "", "git_commit": "", @@ -70,7 +69,7 @@ DEFAULT_LLM_PROVIDER_BINDINGS: Dict[str, str] = { # #region DEFAULT_LLM_ASSISTANT_SETTINGS [C:2] [TYPE Constant] # @BRIEF Default planner settings for assistant chat intent model/provider resolution. -DEFAULT_LLM_ASSISTANT_SETTINGS: Dict[str, str] = { +DEFAULT_LLM_ASSISTANT_SETTINGS: dict[str, str] = { "assistant_planner_provider": "", "assistant_planner_model": "", } @@ -82,8 +81,8 @@ DEFAULT_LLM_ASSISTANT_SETTINGS: Dict[str, str] = { # @PRE: llm_settings is dictionary-like value or None. # @POST: Returned dict contains prompts with all required template keys. # @RELATION DEPENDS_ON -> LLMProviderService -def normalize_llm_settings(llm_settings: Any) -> Dict[str, Any]: - normalized: Dict[str, Any] = { +def normalize_llm_settings(llm_settings: Any) -> dict[str, Any]: + normalized: dict[str, Any] = { "providers": [], "default_provider": "", "prompts": {}, @@ -126,7 +125,7 @@ def normalize_llm_settings(llm_settings: Any) -> Dict[str, Any]: # @PRE: model_name may be empty or mixed-case. # @POST: Returns True when model likely supports multimodal input. # @RELATION DEPENDS_ON -> LLMProviderService -def is_multimodal_model(model_name: str, provider_type: Optional[str] = None) -> bool: +def is_multimodal_model(model_name: str, provider_type: str | None = None) -> bool: token = (model_name or "").strip().lower() if not token: return False @@ -184,7 +183,7 @@ def resolve_bound_provider_id(llm_settings: Any, task_key: str) -> str: # @PRE: template is a string and variables values are already stringifiable. # @POST: Returns rendered prompt text with known placeholders substituted. # @RELATION DEPENDS_ON -> LLMProviderService -def render_prompt(template: str, variables: Dict[str, Any]) -> str: +def render_prompt(template: str, variables: dict[str, Any]) -> str: rendered = template for key, value in variables.items(): rendered = rendered.replace("{" + key + "}", str(value)) diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index 28fba6a0..28fe6c2d 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -4,12 +4,14 @@ # @RELATION DEPENDS_ON -> [LLMProvider] # @RELATION DEPENDS_ON -> [EncryptionManager] # @RELATION DEPENDS_ON -> [LLMProviderConfig] -from typing import List, Optional, TYPE_CHECKING -from sqlalchemy.orm import Session -from ..models.llm import LLMProvider -from ..core.logger import belief_scope, logger -from cryptography.fernet import Fernet import os +from typing import TYPE_CHECKING + +from cryptography.fernet import Fernet +from sqlalchemy.orm import Session + +from ..core.logger import belief_scope, logger +from ..models.llm import LLMProvider if TYPE_CHECKING: from ..plugins.llm_analysis.models import LLMProviderConfig @@ -127,7 +129,7 @@ class LLMProviderService: # @PRE: Database connection must be active. # @POST: Returns list of all LLMProvider records. # @RELATION: [DEPENDS_ON] ->[LLMProvider] - def get_all_providers(self) -> List[LLMProvider]: + def get_all_providers(self) -> list[LLMProvider]: with belief_scope("get_all_providers"): return self.db.query(LLMProvider).all() @@ -138,7 +140,7 @@ class LLMProviderService: # @PRE: provider_id must be a valid string. # @POST: Returns LLMProvider or None if not found. # @RELATION: [DEPENDS_ON] ->[LLMProvider] - def get_provider(self, provider_id: str) -> Optional[LLMProvider]: + def get_provider(self, provider_id: str) -> LLMProvider | None: with belief_scope("get_provider"): return ( self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first() @@ -180,7 +182,7 @@ class LLMProviderService: # @RELATION: [CALLS] ->[encrypt] def update_provider( self, provider_id: str, config: "LLMProviderConfig" - ) -> Optional[LLMProvider]: + ) -> LLMProvider | None: with belief_scope("update_provider"): db_provider = self.get_provider(provider_id) if not db_provider: @@ -227,7 +229,7 @@ class LLMProviderService: # @POST: Returns decrypted API key or None on failure. # @RELATION: [DEPENDS_ON] ->[LLMProvider] # @RELATION: [CALLS] ->[decrypt] - def get_decrypted_api_key(self, provider_id: str) -> Optional[str]: + def get_decrypted_api_key(self, provider_id: str) -> str | None: with belief_scope("get_decrypted_api_key"): db_provider = self.get_provider(provider_id) if not db_provider: @@ -248,7 +250,7 @@ class LLMProviderService: ) return decrypted_key except Exception as e: - logger.error(f"[get_decrypted_api_key] Decryption failed: {str(e)}") + logger.error(f"[get_decrypted_api_key] Decryption failed: {e!s}") return None # endregion get_decrypted_api_key diff --git a/backend/src/services/mapping_service.py b/backend/src/services/mapping_service.py index 8d466482..794ef674 100644 --- a/backend/src/services/mapping_service.py +++ b/backend/src/services/mapping_service.py @@ -11,7 +11,7 @@ # # @INVARIANT: Suggestions are based on database names. -from typing import List, Dict + from ..core.logger import belief_scope from ..core.superset_client import SupersetClient from ..core.utils.matching import suggest_mappings @@ -67,7 +67,7 @@ class MappingService: # @RELATION: CALLS -> suggest_mappings async def get_suggestions( self, source_env_id: str, target_env_id: str - ) -> List[Dict]: + ) -> list[dict]: with belief_scope( "MappingService.get_suggestions", f"source={source_env_id}, target={target_env_id}", diff --git a/backend/src/services/notifications/__tests__/test_notification_service.py b/backend/src/services/notifications/__tests__/test_notification_service.py index 26658581..69bbb120 100644 --- a/backend/src/services/notifications/__tests__/test_notification_service.py +++ b/backend/src/services/notifications/__tests__/test_notification_service.py @@ -2,13 +2,13 @@ # @PURPOSE: Unit tests for NotificationService routing and dispatch logic. # @RELATION: TESTS ->[NotificationService:Class] -import pytest -from unittest.mock import MagicMock, AsyncMock, patch -from datetime import time +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest -from src.models.llm import ValidationRecord, ValidationPolicy -from src.models.profile import UserDashboardPreference from src.models.auth import User +from src.models.llm import ValidationPolicy, ValidationRecord +from src.models.profile import UserDashboardPreference from src.services.notifications.service import NotificationService @@ -47,7 +47,7 @@ async def test_should_notify_fail_only(service): @pytest.mark.asyncio async def test_should_notify_warn_and_fail(service): policy = ValidationPolicy(alert_condition="WARN_AND_FAIL") - + record = ValidationRecord(status="FAIL") assert service._should_notify(record, policy) is True @@ -61,7 +61,7 @@ async def test_should_notify_warn_and_fail(service): @pytest.mark.asyncio async def test_resolve_targets_owner_routing(service, mock_db): record = ValidationRecord(dashboard_id="dash-1", environment_id="env-1") - + user = User(email="user@example.com") pref = UserDashboardPreference( user=user, @@ -69,11 +69,11 @@ async def test_resolve_targets_owner_routing(service, mock_db): notify_on_fail=True, superset_username="user1" ) - + mock_db.query.return_value.filter.return_value.all.return_value = [pref] - + targets = service._resolve_targets(record, None) - + assert ("TELEGRAM", "12345") in targets assert ("SMTP", "user@example.com") in targets @@ -85,7 +85,7 @@ async def test_resolve_targets_custom_channels(service): notify_owners=False, custom_channels=[{"type": "SLACK", "target": "#alerts"}] ) - + targets = service._resolve_targets(record, policy) assert targets == [("SLACK", "#alerts")] @@ -94,7 +94,7 @@ async def test_resolve_targets_custom_channels(service): async def test_dispatch_report_skips_if_no_notify(service): record = ValidationRecord(status="PASS") policy = ValidationPolicy(alert_condition="FAIL_ONLY") - + with patch.object(service, "_resolve_targets") as mock_resolve: await service.dispatch_report(record, policy) mock_resolve.assert_not_called() @@ -103,18 +103,18 @@ async def test_dispatch_report_skips_if_no_notify(service): @pytest.mark.asyncio async def test_dispatch_report_calls_providers(service, mock_db): record = ValidationRecord(id="rec-1", status="FAIL", summary="Bad", issues=[]) - + # Mock providers service._initialize_providers() service._providers["TELEGRAM"] = AsyncMock() service._providers["SMTP"] = AsyncMock() - + # Mock targets with patch.object(service, "_resolve_targets") as mock_resolve: mock_resolve.return_value = [("TELEGRAM", "123"), ("SMTP", "a@b.com")] await service.dispatch_report(record, None) - + service._providers["TELEGRAM"].send.assert_called_once() service._providers["SMTP"].send.assert_called_once() -# endregion test_notification_service \ No newline at end of file +# endregion test_notification_service diff --git a/backend/src/services/notifications/providers.py b/backend/src/services/notifications/providers.py index c68aa07d..7d2be2cb 100644 --- a/backend/src/services/notifications/providers.py +++ b/backend/src/services/notifications/providers.py @@ -16,12 +16,13 @@ # @INVARIANT: Providers must be stateless and resilient to network failures. # @INVARIANT: Sensitive credentials must be handled via encrypted config. -from abc import ABC, abstractmethod -from typing import Any, Dict, Optional -import requests import smtplib -from email.mime.text import MIMEText +from abc import ABC, abstractmethod from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from typing import Any + +import requests from ...core.logger import logger @@ -38,7 +39,7 @@ class NotificationProvider(ABC): target: str, subject: str, body: str, - context: Optional[Dict[str, Any]] = None, + context: dict[str, Any] | None = None, ) -> bool: """ Send a notification to a specific target. @@ -58,7 +59,7 @@ class NotificationProvider(ABC): # @BRIEF Delivers notifications via SMTP. # @RELATION INHERITS -> [NotificationProvider] class SMTPProvider(NotificationProvider): - def __init__(self, config: Dict[str, Any]): + def __init__(self, config: dict[str, Any]): self.host = config.get("host") self.port = int(config.get("port", 587)) self.username = config.get("username") @@ -71,7 +72,7 @@ class SMTPProvider(NotificationProvider): target: str, subject: str, body: str, - context: Optional[Dict[str, Any]] = None, + context: dict[str, Any] | None = None, ) -> bool: try: msg = MIMEMultipart() @@ -102,7 +103,7 @@ class SMTPProvider(NotificationProvider): # @BRIEF Delivers notifications via Telegram Bot API. # @RELATION INHERITS -> [NotificationProvider] class TelegramProvider(NotificationProvider): - def __init__(self, config: Dict[str, Any]): + def __init__(self, config: dict[str, Any]): self.bot_token = config.get("bot_token") async def send( @@ -110,7 +111,7 @@ class TelegramProvider(NotificationProvider): target: str, subject: str, body: str, - context: Optional[Dict[str, Any]] = None, + context: dict[str, Any] | None = None, ) -> bool: if not self.bot_token: logger.error("[TelegramProvider][FAILED] Bot token not configured") @@ -141,7 +142,7 @@ class TelegramProvider(NotificationProvider): # @BRIEF Delivers notifications via Slack Webhooks or API. # @RELATION INHERITS -> [NotificationProvider] class SlackProvider(NotificationProvider): - def __init__(self, config: Dict[str, Any]): + def __init__(self, config: dict[str, Any]): self.webhook_url = config.get("webhook_url") async def send( @@ -149,7 +150,7 @@ class SlackProvider(NotificationProvider): target: str, subject: str, body: str, - context: Optional[Dict[str, Any]] = None, + context: dict[str, Any] | None = None, ) -> bool: if not self.webhook_url: logger.error("[SlackProvider][FAILED] Webhook URL not configured") diff --git a/backend/src/services/notifications/service.py b/backend/src/services/notifications/service.py index 737d9a8f..de48c197 100644 --- a/backend/src/services/notifications/service.py +++ b/backend/src/services/notifications/service.py @@ -16,19 +16,18 @@ # @POST: Notification dispatched via configured providers # @SIDE_EFFECT: Sends notifications via configured providers -from typing import Any, Dict, List, Optional from fastapi import BackgroundTasks from sqlalchemy.orm import Session -from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager -from ...models.llm import ValidationRecord, ValidationPolicy +from ...core.logger import belief_scope, logger +from ...models.llm import ValidationPolicy, ValidationRecord from ...models.profile import UserDashboardPreference from .providers import ( + NotificationProvider, + SlackProvider, SMTPProvider, TelegramProvider, - SlackProvider, - NotificationProvider, ) @@ -50,7 +49,7 @@ class NotificationService: def __init__(self, db: Session, config_manager: ConfigManager): self.db = db self.config_manager = config_manager - self._providers: Dict[str, NotificationProvider] = {} + self._providers: dict[str, NotificationProvider] = {} self._initialized = False # endregion NotificationService_init @@ -93,8 +92,8 @@ class NotificationService: async def dispatch_report( self, record: ValidationRecord, - policy: Optional[ValidationPolicy] = None, - background_tasks: Optional[BackgroundTasks] = None, + policy: ValidationPolicy | None = None, + background_tasks: BackgroundTasks | None = None, ): """ Route a validation record to owners and custom channels. @@ -144,7 +143,7 @@ class NotificationService: # @RELATION: [DEPENDS_ON] ->[ValidationRecord] # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] def _should_notify( - self, record: ValidationRecord, policy: Optional[ValidationPolicy] + self, record: ValidationRecord, policy: ValidationPolicy | None ) -> bool: condition = policy.alert_condition if policy else "FAIL_ONLY" @@ -162,8 +161,8 @@ class NotificationService: # @RELATION: [DEPENDS_ON] ->[ValidationRecord] # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] def _resolve_targets( - self, record: ValidationRecord, policy: Optional[ValidationPolicy] - ) -> List[tuple]: + self, record: ValidationRecord, policy: ValidationPolicy | None + ) -> list[tuple]: targets = [] # Owner routing @@ -198,7 +197,7 @@ class NotificationService: # @RELATION: [DEPENDS_ON] ->[UserDashboardPreference] def _find_dashboard_owners( self, record: ValidationRecord - ) -> List[UserDashboardPreference]: + ) -> list[UserDashboardPreference]: # This is a simplified owner lookup. # In a real scenario, we'd query Superset for owners, then match them to our UserDashboardPreference. # For now, we'll return all users who have bound this dashboard's environment and have a username. diff --git a/backend/src/services/profile_service.py b/backend/src/services/profile_service.py index e9955e56..a010e634 100644 --- a/backend/src/services/profile_service.py +++ b/backend/src/services/profile_service.py @@ -19,31 +19,33 @@ # @TEST_INVARIANT: normalization_consistency -> VERIFIED_BY: [valid_profile_update, enable_without_username] # @DATA_CONTRACT: Profile_id -> ProfileInfo; session_id -> valid UUID # @PRE: Session is active and valid -# @POST: Profile with updated fields populated and +# @POST: Profile with updated fields populated and # @SIDE_EFFECT: Database read/write operations +from collections.abc import Iterable, Sequence from datetime import datetime -from typing import Any, Iterable, List, Optional, Sequence, Set, Tuple +from typing import Any + from sqlalchemy.orm import Session from ..core.auth.repository import AuthRepository -from ..core.logger import logger, belief_scope +from ..core.logger import belief_scope, logger from ..core.superset_client import SupersetClient from ..core.superset_profile_lookup import SupersetAccountLookupAdapter from ..models.auth import User from ..models.profile import UserDashboardPreference -from .llm_provider import EncryptionManager -from .rbac_permission_catalog import discover_declared_permissions from ..schemas.profile import ( ProfilePermissionState, ProfilePreference, ProfilePreferenceResponse, ProfilePreferenceUpdateRequest, ProfileSecuritySummary, + SupersetAccountCandidate, SupersetAccountLookupRequest, SupersetAccountLookupResponse, - SupersetAccountCandidate, ) +from .llm_provider import EncryptionManager +from .rbac_permission_catalog import discover_declared_permissions SUPPORTED_START_PAGES = {"dashboards", "datasets", "reports"} SUPPORTED_DENSITIES = {"compact", "comfortable"} @@ -184,7 +186,7 @@ class ProfileService: self, current_user: User, payload: ProfilePreferenceUpdateRequest, - target_user_id: Optional[str] = None, + target_user_id: str | None = None, ) -> ProfilePreferenceResponse: with belief_scope( "ProfileService.update_my_preference", f"user_id={current_user.id}" @@ -411,9 +413,9 @@ class ProfileService: # @POST: Returns True when normalized username matches owners or modified_by. def matches_dashboard_actor( self, - bound_username: Optional[str], - owners: Optional[Iterable[Any]], - modified_by: Optional[str], + bound_username: str | None, + owners: Iterable[Any] | None, + modified_by: str | None, ) -> bool: normalized_actor = self._normalize_username(bound_username) if not normalized_actor: @@ -436,7 +438,7 @@ class ProfileService: # @PRE: current_user is authenticated. # @POST: Returns deterministic security projection for profile UI. def _build_security_summary(self, current_user: User) -> ProfileSecuritySummary: - role_names_set: Set[str] = set() + role_names_set: set[str] = set() roles = getattr(current_user, "roles", []) or [] for role in roles: normalized_role_name = self._sanitize_text(getattr(role, "name", None)) @@ -447,7 +449,7 @@ class ProfileService: is_admin = any(str(role_name).lower() == "admin" for role_name in role_names) user_permission_pairs = self._collect_user_permission_pairs(current_user) - declared_permission_pairs: Set[Tuple[str, str]] = set() + declared_permission_pairs: set[tuple[str, str]] = set() try: discovered_permissions = discover_declared_permissions( plugin_loader=self.plugin_loader @@ -501,8 +503,8 @@ class ProfileService: # @POST: Returns unique normalized (resource, ACTION) tuples. def _collect_user_permission_pairs( self, current_user: User - ) -> Set[Tuple[str, str]]: - collected: Set[Tuple[str, str]] = set() + ) -> set[tuple[str, str]]: + collected: set[tuple[str, str]] = set() roles = getattr(current_user, "roles", []) or [] for role in roles: permissions = getattr(role, "permissions", []) or [] @@ -594,7 +596,7 @@ class ProfileService: # @PURPOSE: Build a safe display value for sensitive secrets. # @PRE: secret may be None or plaintext. # @POST: Returns masked representation or None. - def _mask_secret_value(self, secret: Optional[str]) -> Optional[str]: + def _mask_secret_value(self, secret: str | None) -> str | None: sanitized_secret = self._sanitize_secret(secret) if sanitized_secret is None: return None @@ -609,7 +611,7 @@ class ProfileService: # @PURPOSE: Normalize optional text into trimmed form or None. # @PRE: value may be empty or None. # @POST: Returns trimmed value or None. - def _sanitize_text(self, value: Optional[str]) -> Optional[str]: + def _sanitize_text(self, value: str | None) -> str | None: normalized = str(value or "").strip() if not normalized: return None @@ -622,7 +624,7 @@ class ProfileService: # @PURPOSE: Normalize secret input into trimmed form or None. # @PRE: value may be None or blank. # @POST: Returns trimmed secret or None. - def _sanitize_secret(self, value: Optional[str]) -> Optional[str]: + def _sanitize_secret(self, value: str | None) -> str | None: if value is None: return None normalized = str(value).strip() @@ -637,7 +639,7 @@ class ProfileService: # @PURPOSE: Normalize supported start page aliases to canonical values. # @PRE: value may be None or alias. # @POST: Returns one of SUPPORTED_START_PAGES. - def _normalize_start_page(self, value: Optional[str]) -> str: + def _normalize_start_page(self, value: str | None) -> str: normalized = str(value or "").strip().lower() if normalized == "reports-logs": return "reports" @@ -652,7 +654,7 @@ class ProfileService: # @PURPOSE: Normalize supported density aliases to canonical values. # @PRE: value may be None or alias. # @POST: Returns one of SUPPORTED_DENSITIES. - def _normalize_density(self, value: Optional[str]) -> str: + def _normalize_density(self, value: str | None) -> str: normalized = str(value or "").strip().lower() if normalized == "free": return "comfortable" @@ -681,7 +683,7 @@ class ProfileService: # @PURPOSE: Return persisted preference row for user or None. # @PRE: user_id is provided. # @POST: Returns matching row or None. - def _get_preference_row(self, user_id: str) -> Optional[UserDashboardPreference]: + def _get_preference_row(self, user_id: str) -> UserDashboardPreference | None: return self.auth_repository.get_user_dashboard_preference(str(user_id)) # endregion _get_preference_row @@ -735,14 +737,14 @@ class ProfileService: # @POST: Returns validation errors list; empty list means valid. def _validate_update_payload( self, - superset_username: Optional[str], + superset_username: str | None, show_only_my_dashboards: bool, - git_email: Optional[str], + git_email: str | None, start_page: str, dashboards_table_density: str, - email_address: Optional[str] = None, - ) -> List[str]: - errors: List[str] = [] + email_address: str | None = None, + ) -> list[str]: + errors: list[str] = [] sanitized_username = self._sanitize_username(superset_username) if sanitized_username and any(ch.isspace() for ch in sanitized_username): @@ -789,7 +791,7 @@ class ProfileService: # @PURPOSE: Normalize raw username into trimmed form or None for empty input. # @PRE: value can be empty or None. # @POST: Returns trimmed username or None. - def _sanitize_username(self, value: Optional[str]) -> Optional[str]: + def _sanitize_username(self, value: str | None) -> str | None: return self._sanitize_text(value) # endregion _sanitize_username @@ -799,7 +801,7 @@ class ProfileService: # @PURPOSE: Apply deterministic trim+lower normalization for actor matching. # @PRE: value can be empty or None. # @POST: Returns lowercase normalized token or None. - def _normalize_username(self, value: Optional[str]) -> Optional[str]: + def _normalize_username(self, value: str | None) -> str | None: sanitized = self._sanitize_username(value) if sanitized is None: return None @@ -812,12 +814,12 @@ class ProfileService: # @PURPOSE: Normalize owners payload into deduplicated lower-cased tokens. # @PRE: owners can be iterable of scalars or dict-like values. # @POST: Returns list of unique normalized owner tokens. - def _normalize_owner_tokens(self, owners: Optional[Iterable[Any]]) -> List[str]: + def _normalize_owner_tokens(self, owners: Iterable[Any] | None) -> list[str]: if owners is None: return [] - normalized: List[str] = [] + normalized: list[str] = [] for owner in owners: - owner_candidates: List[Any] + owner_candidates: list[Any] if isinstance(owner, dict): first_name = self._sanitize_username(str(owner.get("first_name") or "")) last_name = self._sanitize_username(str(owner.get("last_name") or "")) diff --git a/backend/src/services/rbac_permission_catalog.py b/backend/src/services/rbac_permission_catalog.py index 6e573698..b4c8be4b 100644 --- a/backend/src/services/rbac_permission_catalog.py +++ b/backend/src/services/rbac_permission_catalog.py @@ -3,14 +3,15 @@ # @BRIEF Discovers declared RBAC permissions from API routes/plugins and synchronizes them into auth database. import re +from collections.abc import Iterable from functools import lru_cache from pathlib import Path -from typing import Iterable, Set, Tuple from sqlalchemy.orm import Session from ..core.logger import belief_scope, logger from ..models.auth import Permission + # [/SECTION: IMPORTS] # #region HAS_PERMISSION_PATTERN [TYPE Constant] @@ -49,9 +50,9 @@ def _iter_route_files() -> Iterable[Path]: # @BRIEF Extracts explicit has_permission declarations from API route source code. # @PRE: Route files are readable UTF-8 text files. # @POST: Returns unique set of (resource, action) pairs declared in route guards. -def _discover_route_permissions() -> Set[Tuple[str, str]]: +def _discover_route_permissions() -> set[tuple[str, str]]: with belief_scope("rbac_permission_catalog._discover_route_permissions"): - discovered: Set[Tuple[str, str]] = set() + discovered: set[tuple[str, str]] = set() for route_file in _iter_route_files(): try: source = route_file.read_text(encoding="utf-8") @@ -76,7 +77,7 @@ def _discover_route_permissions() -> Set[Tuple[str, str]]: # @PRE: None. # @POST: Returns stable discovered route permission pairs without repeated filesystem scans. @lru_cache(maxsize=1) -def _discover_route_permissions_cached() -> Tuple[Tuple[str, str], ...]: +def _discover_route_permissions_cached() -> tuple[tuple[str, str], ...]: with belief_scope("rbac_permission_catalog._discover_route_permissions_cached"): return tuple(sorted(_discover_route_permissions())) # #endregion _discover_route_permissions_cached @@ -86,9 +87,9 @@ def _discover_route_permissions_cached() -> Tuple[Tuple[str, str], ...]: # @BRIEF Derives dynamic task permissions of form plugin:{plugin_id}:EXECUTE from plugin registry. # @PRE: plugin_loader is optional and may expose get_all_plugin_configs. # @POST: Returns unique plugin EXECUTE permissions if loader is available. -def _discover_plugin_execute_permissions(plugin_loader=None) -> Set[Tuple[str, str]]: +def _discover_plugin_execute_permissions(plugin_loader=None) -> set[tuple[str, str]]: with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions"): - discovered: Set[Tuple[str, str]] = set() + discovered: set[tuple[str, str]] = set() if plugin_loader is None: return discovered @@ -115,8 +116,8 @@ def _discover_plugin_execute_permissions(plugin_loader=None) -> Set[Tuple[str, s # @POST: Returns stable permission tuple without repeated plugin catalog expansion. @lru_cache(maxsize=8) def _discover_plugin_execute_permissions_cached( - plugin_ids: Tuple[str, ...], -) -> Tuple[Tuple[str, str], ...]: + plugin_ids: tuple[str, ...], +) -> tuple[tuple[str, str], ...]: with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions_cached"): return tuple((f"plugin:{plugin_id}", "EXECUTE") for plugin_id in plugin_ids) # #endregion _discover_plugin_execute_permissions_cached @@ -126,7 +127,7 @@ def _discover_plugin_execute_permissions_cached( # @BRIEF Builds canonical RBAC permission catalog from routes and plugin registry. # @PRE: plugin_loader may be provided for dynamic task plugin permission discovery. # @POST: Returns union of route-declared and dynamic plugin EXECUTE permissions. -def discover_declared_permissions(plugin_loader=None) -> Set[Tuple[str, str]]: +def discover_declared_permissions(plugin_loader=None) -> set[tuple[str, str]]: with belief_scope("rbac_permission_catalog.discover_declared_permissions"): permissions = set(_discover_route_permissions_cached()) plugin_ids = tuple( @@ -151,10 +152,10 @@ def discover_declared_permissions(plugin_loader=None) -> Set[Tuple[str, str]]: # @SIDE_EFFECT: Commits auth database transaction when new permissions are added. def sync_permission_catalog( db: Session, - declared_permissions: Iterable[Tuple[str, str]], + declared_permissions: Iterable[tuple[str, str]], ) -> int: with belief_scope("rbac_permission_catalog.sync_permission_catalog"): - normalized_declared: Set[Tuple[str, str]] = set() + normalized_declared: set[tuple[str, str]] = set() for resource, action in declared_permissions: normalized_resource = str(resource or "").strip() normalized_action = str(action or "").strip().upper() diff --git a/backend/src/services/reports/__tests__/test_report_service.py b/backend/src/services/reports/__tests__/test_report_service.py index 90e81a2b..3a4d3405 100644 --- a/backend/src/services/reports/__tests__/test_report_service.py +++ b/backend/src/services/reports/__tests__/test_report_service.py @@ -5,11 +5,11 @@ import sys from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest -from unittest.mock import MagicMock, patch -from datetime import datetime, timezone, timedelta +from datetime import datetime +from unittest.mock import MagicMock # region _make_task [TYPE Function] diff --git a/backend/src/services/reports/__tests__/test_type_profiles.py b/backend/src/services/reports/__tests__/test_type_profiles.py index 1775a6cf..14fe0f9c 100644 --- a/backend/src/services/reports/__tests__/test_type_profiles.py +++ b/backend/src/services/reports/__tests__/test_type_profiles.py @@ -3,9 +3,9 @@ # @PURPOSE: Contract testing for task type profiles and resolution logic. # endregion __tests__/test_report_type_profiles -import pytest from src.models.report import TaskType -from src.services.reports.type_profiles import resolve_task_type, get_type_profile +from src.services.reports.type_profiles import get_type_profile, resolve_task_type + # @TEST_CONTRACT: ResolveTaskType -> Invariants # @TEST_INVARIANT: fallback_to_unknown @@ -58,7 +58,7 @@ def test_get_type_profile_fallback(): profile = get_type_profile(TaskType.UNKNOWN) assert profile["display_label"] == "Other / Unknown" assert profile["fallback"] is True - + # Passing a value that might not be in the dict explicitly profile_fallback = get_type_profile("non-enum-value") assert profile_fallback["display_label"] == "Other / Unknown" diff --git a/backend/src/services/reports/normalizer.py b/backend/src/services/reports/normalizer.py index e01a9bb7..ac3f4dfe 100644 --- a/backend/src/services/reports/normalizer.py +++ b/backend/src/services/reports/normalizer.py @@ -11,7 +11,7 @@ # @SIDE_EFFECT: Read-only database operations from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any from ...core.logger import belief_scope from ...core.task_manager.models import Task, TaskStatus @@ -62,7 +62,7 @@ def build_summary(task: Task, report_status: ReportStatus) -> str: # @BRIEF Extract normalized error context and next actions for failed/partial reports. # @PRE: task is a valid Task object. # @POST: Returns ErrorContext for failed/partial when context exists; otherwise None. -def extract_error_context(task: Task, report_status: ReportStatus) -> Optional[ErrorContext]: +def extract_error_context(task: Task, report_status: ReportStatus) -> ErrorContext | None: with belief_scope("extract_error_context"): if report_status not in {ReportStatus.FAILED, ReportStatus.PARTIAL}: return None @@ -128,7 +128,7 @@ def normalize_task_report(task: Task) -> TaskReport: if not updated_at: updated_at = started_at or datetime.utcnow() - details: Dict[str, Any] = { + details: dict[str, Any] = { "profile": { "display_label": profile.get("display_label"), "visual_variant": profile.get("visual_variant"), @@ -138,7 +138,7 @@ def normalize_task_report(task: Task) -> TaskReport: "result": task.result if task.result is not None else {"note": "Not provided"}, } - source_ref: Dict[str, Any] = {} + source_ref: dict[str, Any] = {} if isinstance(task.params, dict): for key in ("environment_id", "source_env_id", "target_env_id", "dashboard_id", "dataset_id", "resource_id"): if key in task.params: @@ -158,4 +158,4 @@ def normalize_task_report(task: Task) -> TaskReport: ) # #endregion normalize_task_report -# #endregion normalizer \ No newline at end of file +# #endregion normalizer diff --git a/backend/src/services/reports/report_service.py b/backend/src/services/reports/report_service.py index 5e8e70ae..e882f116 100644 --- a/backend/src/services/reports/report_service.py +++ b/backend/src/services/reports/report_service.py @@ -14,11 +14,9 @@ # @POST: Returns Report with generated summary # @SIDE_EFFECT: Read-only database operations; logs report generation -from datetime import datetime, timezone -from typing import List, Optional +from datetime import UTC, datetime from ...core.logger import belief_scope - from ...core.task_manager import TaskManager from ...models.report import ( ReportCollection, @@ -70,7 +68,7 @@ class ReportsService: def __init__( self, task_manager: TaskManager, - clean_release_repository: Optional[CleanReleaseRepository] = None, + clean_release_repository: CleanReleaseRepository | None = None, ): with belief_scope("__init__"): self.task_manager = task_manager @@ -84,7 +82,7 @@ class ReportsService: # @POST: Returns normalized report list preserving source cardinality. # @INVARIANT: Every returned item is a TaskReport. # @RETURN: List[TaskReport] - Reports sorted later by list logic. - def _load_normalized_reports(self) -> List[TaskReport]: + def _load_normalized_reports(self) -> list[TaskReport]: with belief_scope("_load_normalized_reports"): tasks = self.task_manager.get_all_tasks() reports = [normalize_task_report(task) for task in tasks] @@ -99,13 +97,13 @@ class ReportsService: # @INVARIANT: Naive datetimes are interpreted as UTC to preserve deterministic ordering/filtering. # @PARAM: value (Optional[datetime]) - Source datetime value. # @RETURN: Optional[datetime] - UTC-aware datetime or None. - def _to_utc_datetime(self, value: Optional[datetime]) -> Optional[datetime]: + def _to_utc_datetime(self, value: datetime | None) -> datetime | None: with belief_scope("_to_utc_datetime"): if value is None: return None if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) # endregion _to_utc_datetime @@ -173,8 +171,8 @@ class ReportsService: # @PARAM: query (ReportQuery) - Sort config. # @RETURN: List[TaskReport] - Sorted reports. def _sort_reports( - self, reports: List[TaskReport], query: ReportQuery - ) -> List[TaskReport]: + self, reports: list[TaskReport], query: ReportQuery + ) -> list[TaskReport]: with belief_scope("_sort_reports"): reverse = query.sort_order == "desc" @@ -226,7 +224,7 @@ class ReportsService: # @POST: Returns normalized detail envelope with diagnostics and next actions where applicable. # @PARAM: report_id (str) - Stable report identifier. # @RETURN: Optional[ReportDetailView] - Detailed report or None if not found. - def get_report_detail(self, report_id: str) -> Optional[ReportDetailView]: + def get_report_detail(self, report_id: str) -> ReportDetailView | None: with belief_scope("get_report_detail"): reports = self._load_normalized_reports() target = next( diff --git a/backend/src/services/reports/type_profiles.py b/backend/src/services/reports/type_profiles.py index 514d7fea..b038e734 100644 --- a/backend/src/services/reports/type_profiles.py +++ b/backend/src/services/reports/type_profiles.py @@ -1,14 +1,14 @@ # #region type_profiles [C:2] [TYPE Module] [SEMANTICS report, task-type, mapping, profile, fallback] # @BRIEF Deterministic mapping of plugin/task identifiers to canonical report task types and fallback profile metadata. -from typing import Any, Dict, Optional +from typing import Any from ...core.logger import belief_scope from ...models.report import TaskType # #region PLUGIN_TO_TASK_TYPE [TYPE Data] # @BRIEF Maps plugin identifiers to normalized report task types. -PLUGIN_TO_TASK_TYPE: Dict[str, TaskType] = { +PLUGIN_TO_TASK_TYPE: dict[str, TaskType] = { "llm_dashboard_validation": TaskType.LLM_VERIFICATION, "superset-backup": TaskType.BACKUP, "superset-migration": TaskType.MIGRATION, @@ -20,7 +20,7 @@ PLUGIN_TO_TASK_TYPE: Dict[str, TaskType] = { # #region TASK_TYPE_PROFILES [TYPE Data] # @BRIEF Profile metadata registry for each normalized task type. -TASK_TYPE_PROFILES: Dict[TaskType, Dict[str, Any]] = { +TASK_TYPE_PROFILES: dict[TaskType, dict[str, Any]] = { TaskType.LLM_VERIFICATION: { "display_label": "LLM Verification", "visual_variant": "llm", @@ -82,7 +82,7 @@ TASK_TYPE_PROFILES: Dict[TaskType, Dict[str, Any]] = { # @TEST_EDGE: none_plugin -> {"plugin_id": None} # @TEST_EDGE: unknown_plugin -> {"plugin_id": "invalid-plugin"} # @TEST_INVARIANT: fallback_to_unknown -> verifies: [empty_plugin, none_plugin, unknown_plugin] -def resolve_task_type(plugin_id: Optional[str]) -> TaskType: +def resolve_task_type(plugin_id: str | None) -> TaskType: with belief_scope("resolve_task_type"): normalized = (plugin_id or "").strip() if not normalized: @@ -106,7 +106,7 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType: # @TEST_FIXTURE: valid_profile -> {"task_type": "migration"} # @TEST_EDGE: missing_profile -> {"task_type": "some_new_type"} # @TEST_INVARIANT: always_returns_dict -> verifies: [valid_profile, missing_profile] -def get_type_profile(task_type: TaskType) -> Dict[str, Any]: +def get_type_profile(task_type: TaskType) -> dict[str, Any]: with belief_scope("get_type_profile"): return TASK_TYPE_PROFILES.get(task_type, TASK_TYPE_PROFILES[TaskType.UNKNOWN]) diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index 058e01fe..7ede82cf 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -7,19 +7,21 @@ # @RELATION DEPENDS_ON -> [GitService] # @INVARIANT: All resources include metadata about their current state -from typing import List, Dict, Optional, Any -from datetime import datetime, timezone +from datetime import UTC, datetime +from typing import Any + +from ..core.logger import belief_scope, logger from ..core.superset_client import SupersetClient from ..core.task_manager.models import Task from ..services.git_service import GitService -from ..core.logger import logger, belief_scope + # #region ResourceService [C:3] [TYPE Class] # @BRIEF Provides centralized access to resource data with enhanced metadata # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [GitService] class ResourceService: - + # region ResourceService_init [TYPE Function] # @PURPOSE: Initialize the resource service with dependencies # @PRE: None @@ -29,7 +31,7 @@ class ResourceService: self.git_service = GitService() logger.reason("Initialized ResourceService", extra={"src": "ResourceService"}) # endregion ResourceService_init - + # region get_dashboards_with_status [TYPE Function] # @PURPOSE: Fetch dashboards from environment with Git status and last task status # @PRE: env is a valid Environment object @@ -41,30 +43,30 @@ class ResourceService: # @RELATION: CALLS ->[_get_git_status_for_dashboard] # @RELATION: CALLS ->[_get_last_llm_task_for_dashboard] async def get_dashboards_with_status( - self, - env: Any, - tasks: Optional[List[Task]] = None, + self, + env: Any, + tasks: list[Task] | None = None, include_git_status: bool = True, require_slug: bool = False, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: with belief_scope("get_dashboards_with_status", f"env={env.id}"): client = SupersetClient(env) dashboards = client.get_dashboards_summary(require_slug=require_slug) - + # Enhance each dashboard with Git status and task status result = [] for dashboard in dashboards: # dashboard is already a dict, no need to call .dict() dashboard_dict = dashboard dashboard_id = dashboard_dict.get('id') - + # Git status can be skipped for list endpoints and loaded lazily on UI side. if include_git_status: git_status = self._get_git_status_for_dashboard(dashboard_id) dashboard_dict['git_status'] = git_status else: dashboard_dict['git_status'] = None - + # Show status of the latest LLM validation for this dashboard. last_task = self._get_last_llm_task_for_dashboard( dashboard_id, @@ -72,9 +74,9 @@ class ResourceService: tasks, ) dashboard_dict['last_task'] = last_task - + result.append(dashboard_dict) - + logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status") return result # endregion get_dashboards_with_status @@ -94,13 +96,13 @@ class ResourceService: async def get_dashboards_page_with_status( self, env: Any, - tasks: Optional[List[Task]] = None, + tasks: list[Task] | None = None, page: int = 1, page_size: int = 10, - search: Optional[str] = None, + search: str | None = None, include_git_status: bool = True, require_slug: bool = False, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: with belief_scope( "get_dashboards_page_with_status", f"env={env.id}, page={page}, page_size={page_size}, search={search}", @@ -159,9 +161,9 @@ class ResourceService: def _get_last_llm_task_for_dashboard( self, dashboard_id: int, - env_id: Optional[str], - tasks: Optional[List[Task]] = None, - ) -> Optional[Dict[str, Any]]: + env_id: str | None, + tasks: list[Task] | None = None, + ) -> dict[str, Any] | None: if not tasks: return None @@ -252,7 +254,7 @@ class ResourceService: # @PARAM: raw_status (Any) - Raw validation status from task result # @RETURN: Optional[str] - PASS|FAIL|WARN|UNKNOWN # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard] - def _normalize_validation_status(self, raw_status: Any) -> Optional[str]: + def _normalize_validation_status(self, raw_status: Any) -> str | None: if raw_status is None: return None status_text = str(raw_status).strip().upper() @@ -272,11 +274,11 @@ class ResourceService: def _normalize_datetime_for_compare(self, value: Any) -> datetime: if isinstance(value, datetime): if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) - return datetime.min.replace(tzinfo=timezone.utc) + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + return datetime.min.replace(tzinfo=UTC) # endregion _normalize_datetime_for_compare - + # region get_datasets_with_status [TYPE Function] # @PURPOSE: Fetch datasets from environment with mapping progress and last task status # @PRE: env is a valid Environment object @@ -287,34 +289,34 @@ class ResourceService: # @RELATION: CALLS -> [SupersetClientGetDatasetsSummary] # @RELATION: CALLS ->[_get_last_task_for_resource] async def get_datasets_with_status( - self, - env: Any, - tasks: Optional[List[Task]] = None - ) -> List[Dict[str, Any]]: + self, + env: Any, + tasks: list[Task] | None = None + ) -> list[dict[str, Any]]: with belief_scope("get_datasets_with_status", f"env={env.id}"): client = SupersetClient(env) datasets = client.get_datasets_summary() - + # Enhance each dataset with task status result = [] for dataset in datasets: # dataset is already a dict, no need to call .dict() dataset_dict = dataset dataset_id = dataset_dict.get('id') - + # Get last task status last_task = self._get_last_task_for_resource( - f"dataset-{dataset_id}", + f"dataset-{dataset_id}", tasks ) dataset_dict['last_task'] = last_task - + result.append(dataset_dict) - + logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status") return result # endregion get_datasets_with_status - + # region get_activity_summary [TYPE Function] # @PURPOSE: Get summary of active and recent tasks for the activity indicator # @PRE: tasks is a list of Task objects @@ -323,21 +325,21 @@ class ResourceService: # @RETURN: Dict - Activity summary # @RELATION: CALLS ->[_extract_resource_name_from_task] # @RELATION: CALLS ->[_extract_resource_type_from_task] - def get_activity_summary(self, tasks: List[Task]) -> Dict[str, Any]: + def get_activity_summary(self, tasks: list[Task]) -> dict[str, Any]: with belief_scope("get_activity_summary"): # Count active (RUNNING, WAITING_INPUT) tasks active_tasks = [ - t for t in tasks + t for t in tasks if t.status in ['RUNNING', 'WAITING_INPUT'] ] - + # Get recent tasks (last 5) recent_tasks = sorted( - tasks, - key=lambda t: t.created_at, + tasks, + key=lambda t: t.created_at, reverse=True )[:5] - + # Format recent tasks for frontend recent_tasks_formatted = [] for task in recent_tasks: @@ -349,13 +351,13 @@ class ResourceService: 'status': task.status, 'started_at': task.created_at.isoformat() if task.created_at else None }) - + return { 'active_count': len(active_tasks), 'recent_tasks': recent_tasks_formatted } # endregion get_activity_summary - + # region _get_git_status_for_dashboard [TYPE Function] # @PURPOSE: Get Git sync status for a dashboard # @PRE: dashboard_id is a valid integer @@ -363,7 +365,7 @@ class ResourceService: # @PARAM: dashboard_id (int) - The dashboard ID # @RETURN: Optional[Dict] - Git status with branch and sync_status # @RELATION: CALLS ->[get_repo] - def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]: + def _get_git_status_for_dashboard(self, dashboard_id: int) -> dict[str, Any] | None: try: repo = self.git_service.get_repo(dashboard_id) if not repo: @@ -373,24 +375,24 @@ class ResourceService: 'has_repo': False, 'has_changes_for_commit': False } - + # Check if there are uncommitted changes try: # Get current branch branch = repo.active_branch.name - + # Check for uncommitted changes is_dirty = repo.is_dirty() has_changes_for_commit = repo.is_dirty(untracked_files=True) - + # Check for unpushed commits unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0 - + if is_dirty or unpushed > 0: sync_status = 'DIFF' else: sync_status = 'OK' - + return { 'branch': branch, 'sync_status': sync_status, @@ -414,7 +416,7 @@ class ResourceService: 'has_changes_for_commit': False } # endregion _get_git_status_for_dashboard - + # region _get_last_task_for_resource [TYPE Function] # @PURPOSE: Get the most recent task for a specific resource # @PRE: resource_id is a valid string @@ -426,33 +428,33 @@ class ResourceService: def _get_last_task_for_resource( self, resource_id: str, - tasks: Optional[List[Task]] = None - ) -> Optional[Dict[str, Any]]: + tasks: list[Task] | None = None + ) -> dict[str, Any] | None: if not tasks: return None - + # Filter tasks for this resource resource_tasks = [] for task in tasks: params = task.params or {} if params.get('resource_id') == resource_id: resource_tasks.append(task) - + if not resource_tasks: return None - + # Get most recent task with timezone-safe comparison. last_task = max( resource_tasks, key=lambda t: self._normalize_datetime_for_compare(getattr(t, "created_at", None)), ) - + return { 'task_id': str(last_task.id), 'status': last_task.status } # endregion _get_last_task_for_resource - + # region _extract_resource_name_from_task [TYPE Function] # @PURPOSE: Extract resource name from task params # @PRE: task is a valid Task object @@ -464,7 +466,7 @@ class ResourceService: params = task.params or {} return params.get('resource_name', f"Task {task.id}") # endregion _extract_resource_name_from_task - + # region _extract_resource_type_from_task [TYPE Function] # @PURPOSE: Extract resource type from task params # @PRE: task is a valid Task object diff --git a/backend/test_pat_api.py b/backend/test_pat_api.py index 7a3a0645..062219c8 100644 --- a/backend/test_pat_api.py +++ b/backend/test_pat_api.py @@ -1,5 +1,6 @@ + from pydantic import BaseModel, Field -from typing import Optional + class GitProvider(str): pass @@ -10,8 +11,8 @@ class GitServerConfigBase(BaseModel): url: str = Field(..., description="Server base URL") pat: str = Field(..., description="Personal Access Token") pat: str = Field(..., description="Personal Access Token") - default_repository: Optional[str] = Field(None, description="Default repository path (org/repo)") - default_branch: Optional[str] = Field("main", description="Default branch logic/name") + default_repository: str | None = Field(None, description="Default repository path (org/repo)") + default_branch: str | None = Field("main", description="Default branch logic/name") class GitServerConfigSchema(GitServerConfigBase): id: str diff --git a/backend/tests/core/test_defensive_guards.py b/backend/tests/core/test_defensive_guards.py index 39a9d2cc..38069df3 100644 --- a/backend/tests/core/test_defensive_guards.py +++ b/backend/tests/core/test_defensive_guards.py @@ -1,15 +1,17 @@ +import shutil import sys from pathlib import Path -import shutil -import pytest from unittest.mock import MagicMock, patch + +import pytest from git.exc import InvalidGitRepositoryError sys.path.insert(0, str(Path(__file__).parent.parent.parent)) -from src.services.git_service import GitService -from src.core.superset_client import SupersetClient from src.core.config_models import Environment +from src.core.superset_client import SupersetClient +from src.services.git_service import GitService + # [DEF:test_git_service_get_repo_path_guard:Function] # @RELATION: BINDS_TO -> UnknownModule diff --git a/backend/tests/core/test_git_service_gitea_pr.py b/backend/tests/core/test_git_service_gitea_pr.py index 4387cf17..492a86f0 100644 --- a/backend/tests/core/test_git_service_gitea_pr.py +++ b/backend/tests/core/test_git_service_gitea_pr.py @@ -9,8 +9,8 @@ import asyncio import sys from pathlib import Path -from fastapi import HTTPException import pytest +from fastapi import HTTPException sys.path.insert(0, str(Path(__file__).parent.parent.parent)) diff --git a/backend/tests/core/test_mapping_service.py b/backend/tests/core/test_mapping_service.py index 30d85669..1775a4d8 100644 --- a/backend/tests/core/test_mapping_service.py +++ b/backend/tests/core/test_mapping_service.py @@ -4,21 +4,21 @@ # @LAYER: Domain # @RELATION: VERIFIES ->[src.core.mapping_service.IdMappingService] # +import sys +from datetime import UTC, datetime +from pathlib import Path + import pytest -from datetime import datetime, timezone from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import sys -import os -from pathlib import Path # Add backend directory to sys.path so 'src' can be resolved backend_dir = str(Path(__file__).parent.parent.parent.resolve()) if backend_dir not in sys.path: sys.path.insert(0, backend_dir) -from src.models.mapping import Base, ResourceMapping, ResourceType from src.core.mapping_service import IdMappingService +from src.models.mapping import Base, ResourceMapping, ResourceType @pytest.fixture @@ -80,7 +80,7 @@ def test_get_remote_id_returns_integer(db_session): uuid="uuid-1", remote_integer_id="99", resource_name="Test DS", - last_synced_at=datetime.now(timezone.utc), + last_synced_at=datetime.now(UTC), ) db_session.add(mapping) db_session.commit() diff --git a/backend/tests/core/test_migration_engine.py b/backend/tests/core/test_migration_engine.py index e35d222b..f76f1a31 100644 --- a/backend/tests/core/test_migration_engine.py +++ b/backend/tests/core/test_migration_engine.py @@ -4,24 +4,21 @@ # @LAYER: Domain # @RELATION: VERIFIES -> [src.core.migration_engine:Module] # -import pytest -import tempfile import json -import yaml -import zipfile -import sys import os +import sys +import tempfile +import zipfile from pathlib import Path -from unittest.mock import MagicMock + +import pytest +import yaml backend_dir = str(Path(__file__).parent.parent.parent.resolve()) if backend_dir not in sys.path: sys.path.insert(0, backend_dir) from src.core.migration_engine import MigrationEngine -from src.core.mapping_service import IdMappingService -from src.models.mapping import ResourceType - # --- Fixtures --- @@ -80,7 +77,7 @@ def test_patch_dashboard_metadata_replaces_chart_ids(): engine._patch_dashboard_metadata(fp, "target-env", source_map) - with open(fp, "r") as f: + with open(fp) as f: data = yaml.safe_load(f) result = json.loads(data["json_metadata"]) assert ( @@ -107,7 +104,7 @@ def test_patch_dashboard_metadata_replaces_dataset_ids(): engine._patch_dashboard_metadata(fp, "target-env", source_map) - with open(fp, "r") as f: + with open(fp) as f: data = yaml.safe_load(f) result = json.loads(data["json_metadata"]) assert ( @@ -134,7 +131,7 @@ def test_patch_dashboard_metadata_skips_when_no_metadata(): engine._patch_dashboard_metadata(fp, "target-env", {}) - with open(fp, "r") as f: + with open(fp) as f: data = yaml.safe_load(f) assert "json_metadata" not in data @@ -162,7 +159,7 @@ def test_patch_dashboard_metadata_handles_missing_targets(): engine._patch_dashboard_metadata(fp, "target-env", source_map) - with open(fp, "r") as f: + with open(fp) as f: data = yaml.safe_load(f) result = json.loads(data["json_metadata"]) targets = result["native_filter_configuration"][0]["targets"] @@ -219,7 +216,7 @@ def test_transform_yaml_replaces_database_uuid(): engine._transform_yaml(fp, {"source-uuid-abc": "target-uuid-xyz"}) - with open(fp, "r") as f: + with open(fp) as f: data = yaml.safe_load(f) assert data["database_uuid"] == "target-uuid-xyz" assert data["table_name"] == "my_table" @@ -243,7 +240,7 @@ def test_transform_yaml_ignores_unmapped_uuid(): engine._transform_yaml(fp, {"other-uuid": "replacement"}) - with open(fp, "r") as f: + with open(fp) as f: data = yaml.safe_load(f) assert data["database_uuid"] == "unknown-uuid" @@ -317,12 +314,12 @@ def test_transform_zip_end_to_end(): out_path = Path(out_dir) # Verify dataset transformation - with open(out_path / "datasets" / "ds.yaml", "r") as f: + with open(out_path / "datasets" / "ds.yaml") as f: ds_data = yaml.safe_load(f) assert ds_data["database_uuid"] == "target-db-uuid" # Verify dashboard patching - with open(out_path / "dashboards" / "db.yaml", "r") as f: + with open(out_path / "dashboards" / "db.yaml") as f: db_data = yaml.safe_load(f) meta = json.loads(db_data["json_metadata"]) assert ( diff --git a/backend/tests/scripts/test_clean_release_cli.py b/backend/tests/scripts/test_clean_release_cli.py index ed58a2f6..c18dadee 100644 --- a/backend/tests/scripts/test_clean_release_cli.py +++ b/backend/tests/scripts/test_clean_release_cli.py @@ -5,21 +5,19 @@ """Smoke tests for the redesigned clean release CLI commands.""" +from datetime import UTC, datetime from types import SimpleNamespace -import json - -from src.dependencies import get_clean_release_repository, get_config_manager -from datetime import datetime, timezone from uuid import uuid4 +from src.dependencies import get_clean_release_repository, get_config_manager from src.models.clean_release import ( CleanPolicySnapshot, ComplianceReport, ReleaseCandidate, SourceRegistrySnapshot, ) -from src.services.clean_release.enums import CandidateStatus, ComplianceDecision from src.scripts.clean_release_cli import main as cli_main +from src.services.clean_release.enums import CandidateStatus, ComplianceDecision # [DEF:test_cli_candidate_register_scaffold:Function] @@ -227,7 +225,7 @@ def test_cli_release_gate_commands_scaffold() -> None: version="1.0.0", source_snapshot_ref="git:sha-approved", created_by="cli-test", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.CHECK_PASSED.value, ) ) @@ -237,7 +235,7 @@ def test_cli_release_gate_commands_scaffold() -> None: version="1.0.0", source_snapshot_ref="git:sha-rejected", created_by="cli-test", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.CHECK_PASSED.value, ) ) @@ -252,7 +250,7 @@ def test_cli_release_gate_commands_scaffold() -> None: "violations_count": 0, "blocking_violations_count": 0, }, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) ) @@ -267,7 +265,7 @@ def test_cli_release_gate_commands_scaffold() -> None: "violations_count": 0, "blocking_violations_count": 0, }, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) ) diff --git a/backend/tests/scripts/test_clean_release_tui.py b/backend/tests/scripts/test_clean_release_tui.py index 17637cc1..100fd99d 100644 --- a/backend/tests/scripts/test_clean_release_tui.py +++ b/backend/tests/scripts/test_clean_release_tui.py @@ -5,17 +5,16 @@ # @LAYER: Scripts # @INVARIANT: TUI initializes, handles hotkeys (F5, F10) and safely falls back without TTY. -import os -import sys import curses import json +import os from unittest import mock from unittest.mock import MagicMock, patch import pytest -from src.scripts.clean_release_tui import CleanReleaseTUI, main, tui_main from src.models.clean_release import CheckFinalStatus +from src.scripts.clean_release_tui import CleanReleaseTUI, main @pytest.fixture @@ -63,17 +62,17 @@ def test_tui_initial_render(mock_curses_module, mock_stdscr: MagicMock): app = CleanReleaseTUI(mock_stdscr) assert app.status == "READY" - + # We only want to run one loop iteration, so we mock getch to return F10 mock_stdscr.getch.return_value = curses.KEY_F10 - + app.loop() - + # Assert header was drawn addstr_calls = mock_stdscr.addstr.call_args_list assert any("Enterprise Clean Release Validator" in str(call) for call in addstr_calls) assert any("Candidate: [2026.03.03-rc1]" in str(call) for call in addstr_calls) - + # Assert checks list is shown assert any("Data Purity" in str(call) for call in addstr_calls) assert any("Internal Sources Only" in str(call) for call in addstr_calls) @@ -98,7 +97,7 @@ def test_tui_run_checks_f5(mock_curses_module, mock_stdscr: MagicMock): mock_curses_module.A_BOLD = 0 app = CleanReleaseTUI(mock_stdscr) - + # getch sequence: # 1. First loop: F5 (triggers run_checks) # 2. Next call after run_checks: F10 to exit @@ -108,12 +107,12 @@ def test_tui_run_checks_f5(mock_curses_module, mock_stdscr: MagicMock): mock_stdscr.f5_pressed = True return curses.KEY_F5 return curses.KEY_F10 - + mock_stdscr.getch.side_effect = side_effect - + with mock.patch("time.sleep", return_value=None): app.loop() - + # After F5 is pressed, status should be BLOCKED due to deliberate 'test-data' violation assert app.status == CheckFinalStatus.BLOCKED assert app.report_id is not None @@ -132,13 +131,13 @@ def test_tui_exit_f10(mock_curses_module, mock_stdscr: MagicMock): """ # Ensure constants match mock_curses_module.KEY_F10 = curses.KEY_F10 - + app = CleanReleaseTUI(mock_stdscr) mock_stdscr.getch.return_value = curses.KEY_F10 - + # loop() should return cleanly app.loop() - + assert app.status == "READY" @@ -159,12 +158,12 @@ def test_tui_clear_history_f7(mock_curses_module, mock_stdscr: MagicMock): app = CleanReleaseTUI(mock_stdscr) app.status = CheckFinalStatus.BLOCKED app.report_id = "SOME-REPORT" - + # F7 then F10 mock_stdscr.getch.side_effect = [curses.KEY_F7, curses.KEY_F10] - + app.loop() - + assert app.status == "READY" assert app.report_id is None assert len(app.checks_progress) == 0 diff --git a/backend/tests/services/clean_release/test_approval_service.py b/backend/tests/services/clean_release/test_approval_service.py index f4d7afb5..5f3e7a43 100644 --- a/backend/tests/services/clean_release/test_approval_service.py +++ b/backend/tests/services/clean_release/test_approval_service.py @@ -7,7 +7,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime import pytest @@ -35,7 +35,7 @@ def _seed_candidate_with_report( version="1.0.0", source_snapshot_ref="git:sha-approve-1", created_by="tester", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.CHECK_PASSED.value, ) ) @@ -50,7 +50,7 @@ def _seed_candidate_with_report( "violations_count": 0, "blocking_violations_count": 0 if report_status == ComplianceDecision.PASSED else 1, }, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) ) @@ -96,7 +96,7 @@ def test_approve_rejects_foreign_report(): candidate_id="cand-foreign-1", final_status=ComplianceDecision.PASSED.value, summary_json={"operator_summary": "foreign", "violations_count": 0, "blocking_violations_count": 0}, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) repository.save_report(foreign_report) @@ -175,8 +175,8 @@ def test_reject_persists_decision_without_promoting_candidate_state(): # @POST: publish_candidate raises PublicationGateError. def test_reject_then_publish_is_blocked(): from src.services.clean_release.approval_service import reject_candidate - from src.services.clean_release.publication_service import publish_candidate from src.services.clean_release.exceptions import PublicationGateError + from src.services.clean_release.publication_service import publish_candidate repository, candidate_id, report_id = _seed_candidate_with_report() @@ -199,4 +199,4 @@ def test_reject_then_publish_is_blocked(): ) # [/DEF:test_reject_then_publish_is_blocked:Function] -# [/DEF:TestApprovalService:Module] \ No newline at end of file +# [/DEF:TestApprovalService:Module] diff --git a/backend/tests/services/clean_release/test_candidate_manifest_services.py b/backend/tests/services/clean_release/test_candidate_manifest_services.py index 7de8b969..87ee70e7 100644 --- a/backend/tests/services/clean_release/test_candidate_manifest_services.py +++ b/backend/tests/services/clean_release/test_candidate_manifest_services.py @@ -3,18 +3,19 @@ # @PURPOSE: Test lifecycle and manifest versioning for release candidates. # @LAYER: Tests +from datetime import UTC, datetime + import pytest -from datetime import datetime, timezone from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker + from src.core.database import Base from src.models.clean_release import ( - ReleaseCandidate, DistributionManifest, - CandidateArtifact, + ReleaseCandidate, ) -from src.services.clean_release.enums import CandidateStatus from src.services.clean_release.candidate_service import register_candidate +from src.services.clean_release.enums import CandidateStatus from src.services.clean_release.manifest_service import build_manifest_snapshot from src.services.clean_release.repository import CleanReleaseRepository @@ -79,7 +80,7 @@ def test_manifest_versioning_and_immutability(db_session): artifacts_digest="hash1", source_snapshot_ref="ref1", content_json={}, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), created_by="operator", ) db_session.add(m1) @@ -93,7 +94,7 @@ def test_manifest_versioning_and_immutability(db_session): artifacts_digest="hash2", source_snapshot_ref="ref1", content_json={}, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), created_by="operator", ) db_session.add(m2) diff --git a/backend/tests/services/clean_release/test_compliance_execution_service.py b/backend/tests/services/clean_release/test_compliance_execution_service.py index 77c1c4cb..1e29a5c5 100644 --- a/backend/tests/services/clean_release/test_compliance_execution_service.py +++ b/backend/tests/services/clean_release/test_compliance_execution_service.py @@ -7,7 +7,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime import pytest @@ -46,7 +46,7 @@ def _seed_with_candidate_policy_registry( version="1.0.0", source_snapshot_ref="git:sha-us2", created_by="tester", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.MANIFEST_BUILT.value, ) ) @@ -89,7 +89,7 @@ def _seed_with_candidate_policy_registry( } }, created_by="tester", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), immutable=True, ) ) diff --git a/backend/tests/services/clean_release/test_compliance_task_integration.py b/backend/tests/services/clean_release/test_compliance_task_integration.py index 7325b5d2..e23ab508 100644 --- a/backend/tests/services/clean_release/test_compliance_task_integration.py +++ b/backend/tests/services/clean_release/test_compliance_task_integration.py @@ -8,8 +8,8 @@ from __future__ import annotations import asyncio -from datetime import datetime, timezone -from typing import Any, Dict +from datetime import UTC, datetime +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -48,7 +48,7 @@ def _seed_repository( version="1.0.0", source_snapshot_ref="git:sha-task-int", created_by="tester", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=CandidateStatus.MANIFEST_BUILT.value, ) ) @@ -91,7 +91,7 @@ def _seed_repository( } }, created_by="tester", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), immutable=True, ) ) @@ -114,7 +114,7 @@ class CleanReleaseCompliancePlugin: def name(self) -> str: return "clean_release_compliance" - def execute(self, params: Dict[str, Any], context=None): + def execute(self, params: dict[str, Any], context=None): orchestrator = CleanComplianceOrchestrator(params["repository"]) run = orchestrator.start_check_run( candidate_id=params["candidate_id"], diff --git a/backend/tests/services/clean_release/test_demo_mode_isolation.py b/backend/tests/services/clean_release/test_demo_mode_isolation.py index b87a5b56..1e1e9864 100644 --- a/backend/tests/services/clean_release/test_demo_mode_isolation.py +++ b/backend/tests/services/clean_release/test_demo_mode_isolation.py @@ -6,7 +6,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from src.models.clean_release import ReleaseCandidate from src.services.clean_release.demo_data_service import ( @@ -65,7 +65,7 @@ def test_create_isolated_repository_keeps_mode_data_separate() -> None: version="1.0.0", source_snapshot_ref="git:sha-demo", created_by="demo-operator", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status="DRAFT", ) ) @@ -75,7 +75,7 @@ def test_create_isolated_repository_keeps_mode_data_separate() -> None: version="1.0.0", source_snapshot_ref="git:sha-real", created_by="real-operator", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status="DRAFT", ) ) diff --git a/backend/tests/services/clean_release/test_publication_service.py b/backend/tests/services/clean_release/test_publication_service.py index 315fa3a7..304377b1 100644 --- a/backend/tests/services/clean_release/test_publication_service.py +++ b/backend/tests/services/clean_release/test_publication_service.py @@ -7,7 +7,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime import pytest @@ -35,7 +35,7 @@ def _seed_candidate_with_passed_report( version="1.0.0", source_snapshot_ref="git:sha-publish-1", created_by="tester", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), status=candidate_status.value, ) ) @@ -46,7 +46,7 @@ def _seed_candidate_with_passed_report( candidate_id=candidate_id, final_status=ComplianceDecision.PASSED.value, summary_json={"operator_summary": "seed", "violations_count": 0, "blocking_violations_count": 0}, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) ) @@ -146,4 +146,4 @@ def test_republish_after_revoke_creates_new_active_record(): assert second.status == PublicationStatus.ACTIVE.value # [/DEF:test_republish_after_revoke_creates_new_active_record:Function] -# [/DEF:TestPublicationService:Module] \ No newline at end of file +# [/DEF:TestPublicationService:Module] diff --git a/backend/tests/services/clean_release/test_report_audit_immutability.py b/backend/tests/services/clean_release/test_report_audit_immutability.py index 7b563d8b..3af24024 100644 --- a/backend/tests/services/clean_release/test_report_audit_immutability.py +++ b/backend/tests/services/clean_release/test_report_audit_immutability.py @@ -7,7 +7,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from unittest.mock import patch import pytest @@ -15,13 +15,11 @@ import pytest from src.models.clean_release import ( ComplianceReport, ComplianceRun, - ComplianceViolation, ) from src.services.clean_release.audit_service import ( audit_check_run, audit_preparation, audit_report, - audit_violation, ) from src.services.clean_release.enums import ComplianceDecision, RunStatus from src.services.clean_release.report_builder import ComplianceReportBuilder @@ -44,9 +42,9 @@ def _terminal_run( policy_snapshot_id="policy-immut-1", registry_snapshot_id="registry-immut-1", requested_by="tester", - requested_at=datetime.now(timezone.utc), - started_at=datetime.now(timezone.utc), - finished_at=datetime.now(timezone.utc), + requested_at=datetime.now(UTC), + started_at=datetime.now(UTC), + finished_at=datetime.now(UTC), status=RunStatus.SUCCEEDED, final_status=final_status, ) @@ -93,7 +91,7 @@ def test_repository_rejects_report_overwrite_for_same_report_id(): "violations_count": 0, "blocking_violations_count": 0, }, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) mutated = ComplianceReport( @@ -106,7 +104,7 @@ def test_repository_rejects_report_overwrite_for_same_report_id(): "violations_count": 1, "blocking_violations_count": 1, }, - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), immutable=True, ) diff --git a/backend/tests/services/dataset_review/test_superset_matrix.py b/backend/tests/services/dataset_review/test_superset_matrix.py index a2358164..3a1fd3d6 100644 --- a/backend/tests/services/dataset_review/test_superset_matrix.py +++ b/backend/tests/services/dataset_review/test_superset_matrix.py @@ -8,16 +8,14 @@ from types import SimpleNamespace from unittest.mock import MagicMock -# Import models to ensure proper SQLAlchemy registration -from src.models.auth import User -from src.models.dataset_review import CompiledPreview - from src.core.utils.superset_compilation_adapter import ( PreviewCompilationPayload, SqlLabLaunchPayload, SupersetCompilationAdapter, ) +# Import models to ensure proper SQLAlchemy registration + # [DEF:make_adapter:Function] # @PURPOSE: Build an adapter with a mock Superset client and deterministic environment for compatibility tests. diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 1cf0491c..fa83102d 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -13,16 +13,17 @@ from pathlib import Path sys.path.append(str(Path(__file__).parent.parent / "src")) import pytest +from cryptography.fernet import Fernet from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from cryptography.fernet import Fernet -from src.core.database import Base -from src.models.auth import User, Role, Permission, ADGroupMapping -from src.services.auth_service import AuthService + from src.core.auth.repository import AuthRepository -from src.core.auth.security import verify_password, get_password_hash +from src.core.auth.security import get_password_hash, verify_password +from src.core.database import Base +from src.models.auth import ADGroupMapping, Permission, Role, User from src.scripts.create_admin import create_admin from src.scripts.init_auth_db import ensure_encryption_key +from src.services.auth_service import AuthService # Create in-memory SQLite database for testing SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" diff --git a/backend/tests/test_core_scheduler.py b/backend/tests/test_core_scheduler.py index dcb59476..f04d52d8 100644 --- a/backend/tests/test_core_scheduler.py +++ b/backend/tests/test_core_scheduler.py @@ -10,11 +10,9 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest from unittest.mock import MagicMock, patch -from datetime import datetime, timezone -from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.schedulers.background import BackgroundScheduler # --------------------------------------------------------------------------- # Helpers — fresh scheduler & mock config diff --git a/backend/tests/test_dashboards_api.py b/backend/tests/test_dashboards_api.py index a53a7981..57206e72 100644 --- a/backend/tests/test_dashboards_api.py +++ b/backend/tests/test_dashboards_api.py @@ -3,24 +3,26 @@ # @PURPOSE: Comprehensive contract-driven tests for Dashboard Hub API # @LAYER: Domain (Tests) # @SEMANTICS: tests, dashboards, api, contract, remediation +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock, patch + import pytest from fastapi.testclient import TestClient -from unittest.mock import MagicMock, patch, AsyncMock -from datetime import datetime, timezone -from src.app import app + from src.api.routes.dashboards import ( - DashboardsResponse, DashboardDetailResponse, + DashboardsResponse, DashboardTaskHistoryResponse, DatabaseMappingsResponse, ) +from src.app import app from src.dependencies import ( - get_current_user, - has_permission, get_config_manager, - get_task_manager, - get_resource_service, + get_current_user, get_mapping_service, + get_resource_service, + get_task_manager, + has_permission, ) # Global mock user @@ -299,7 +301,7 @@ def test_get_dashboard_detail_env_not_found(mock_deps): # [DEF:test_get_dashboard_tasks_history_success:Function] # @RELATION: BINDS_TO ->[TestDashboardsApi] def test_get_dashboard_tasks_history_success(mock_deps): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) task1 = MagicMock( id="t1", plugin_id="superset-backup", @@ -328,7 +330,7 @@ def test_get_dashboard_tasks_history_sorting(mock_deps): """@POST: Response contains sorted task history (newest first).""" from datetime import timedelta - now = datetime.now(timezone.utc) + now = datetime.now(UTC) older = now - timedelta(hours=2) newest = now diff --git a/backend/tests/test_log_persistence.py b/backend/tests/test_log_persistence.py index a3f7e349..f78f7ac9 100644 --- a/backend/tests/test_log_persistence.py +++ b/backend/tests/test_log_persistence.py @@ -11,9 +11,10 @@ from unittest.mock import patch from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from src.models.mapping import Base -from src.core.task_manager.persistence import TaskLogPersistenceService from src.core.task_manager.models import LogEntry, LogFilter +from src.core.task_manager.persistence import TaskLogPersistenceService +from src.models.mapping import Base + # [/SECTION] # [DEF:TestLogPersistence:Class] diff --git a/backend/tests/test_logger.py b/backend/tests/test_logger.py index ce00cc5a..a3559abc 100644 --- a/backend/tests/test_logger.py +++ b/backend/tests/test_logger.py @@ -5,18 +5,19 @@ # @RELATION: VERIFIES -> src/core/logger.py # @INVARIANT: All required log statements must correctly check the threshold. -import pytest import logging +import pytest + +from src.core.config_models import LoggingConfig from src.core.logger import ( + CotJsonFormatter, belief_scope, - logger, configure_logger, get_task_log_level, + logger, should_log_task_level, - CotJsonFormatter, ) -from src.core.config_models import LoggingConfig @pytest.fixture(autouse=True) @@ -101,9 +102,8 @@ def test_belief_scope_error_handling(caplog): caplog.set_level("DEBUG") - with pytest.raises(ValueError): - with belief_scope("FailingFunction"): - raise ValueError("Something went wrong") + with pytest.raises(ValueError), belief_scope("FailingFunction"): + raise ValueError("Something went wrong") # Check that an EXPLORE marker was emitted explore_records = [ @@ -294,7 +294,6 @@ def test_enable_belief_state_flag(caplog): def test_cot_json_formatter_output(): """Test that CotJsonFormatter produces valid JSON with expected fields.""" import json - from datetime import datetime, timezone formatter = CotJsonFormatter() diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 82494419..a846d202 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -1,6 +1,7 @@ from src.core.config_models import Environment from src.core.logger import belief_scope + # [DEF:test_environment_model:Function] # @RELATION: TESTS -> Environment # @PURPOSE: Tests that Environment model correctly stores values. diff --git a/backend/tests/test_resource_hubs.py b/backend/tests/test_resource_hubs.py index 8f047569..80e2a49e 100644 --- a/backend/tests/test_resource_hubs.py +++ b/backend/tests/test_resource_hubs.py @@ -4,14 +4,16 @@ # @SEMANTICS: tests, resource-hubs, dashboards, datasets, pagination, api # @PURPOSE: Contract tests for resource hub dashboards/datasets listing and pagination boundary validation. # @LAYER: Domain (Tests) +from unittest.mock import AsyncMock, MagicMock + import pytest from fastapi.testclient import TestClient -from unittest.mock import MagicMock, AsyncMock + from src.app import app from src.dependencies import ( get_config_manager, - get_task_manager, get_resource_service, + get_task_manager, has_permission, ) diff --git a/backend/tests/test_smoke_app.py b/backend/tests/test_smoke_app.py index f2e12fa7..10041e08 100644 --- a/backend/tests/test_smoke_app.py +++ b/backend/tests/test_smoke_app.py @@ -64,11 +64,11 @@ def test_app_has_expected_middleware(): def test_cot_logger_imports(): """Core logging infrastructure is importable.""" from src.core.cot_logger import ( - log, - seed_trace_id, get_trace_id, - push_span, + log, pop_span, + push_span, + seed_trace_id, ) tid = seed_trace_id() diff --git a/backend/tests/test_smoke_plugins.py b/backend/tests/test_smoke_plugins.py index fb3d0a09..c99c396c 100644 --- a/backend/tests/test_smoke_plugins.py +++ b/backend/tests/test_smoke_plugins.py @@ -1,9 +1,10 @@ +import os import sys from pathlib import Path -import os -import pytest from unittest.mock import MagicMock, patch +import pytest + sys.path.insert(0, str(Path(__file__).parent.parent)) # Mock database before any modules that import it are loaded @@ -22,18 +23,18 @@ class TestPluginSmoke: syntax errors, missing class declarations). """ from src.core.plugin_loader import PluginLoader - + plugin_dir = os.path.join(str(Path(__file__).parent.parent), "src", "plugins") - + # This will discover and instantiate plugins loader = PluginLoader(plugin_dir) plugins = loader.get_all_plugin_configs() plugin_ids = {p.id for p in plugins} - + # We expect at least the migration and git plugins to be present expected_plugins = {"superset-migration", "git-integration"} - + missing_plugins = expected_plugins - plugin_ids assert not missing_plugins, f"Missing expected plugins: {missing_plugins}" @@ -44,21 +45,21 @@ class TestPluginSmoke: """ from src.core.plugin_loader import PluginLoader from src.core.task_manager.manager import TaskManager - + plugin_dir = os.path.join(str(Path(__file__).parent.parent), "src", "plugins") loader = PluginLoader(plugin_dir) - + # Initialize TaskManager with real loader with patch("src.core.task_manager.manager.TaskPersistenceService") as MockPersistence, \ patch("src.core.task_manager.manager.TaskLogPersistenceService"): - + MockPersistence.return_value.load_tasks.return_value = [] - + with patch("src.dependencies.config_manager"): manager = TaskManager(loader) - + # Stop the flusher thread to prevent hanging manager._flusher_stop_event.set() manager._flusher_thread.join(timeout=2) - + assert manager is not None diff --git a/backend/tests/test_task_manager.py b/backend/tests/test_task_manager.py index fd58af98..5b4586b6 100644 --- a/backend/tests/test_task_manager.py +++ b/backend/tests/test_task_manager.py @@ -7,12 +7,14 @@ import sys from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest import asyncio -from unittest.mock import MagicMock, patch, AsyncMock from datetime import datetime +from unittest.mock import MagicMock, patch + +import pytest # Helper to create a TaskManager with mocked dependencies diff --git a/backend/tests/test_task_persistence.py b/backend/tests/test_task_persistence.py index 93e6c76d..b14bd5d2 100644 --- a/backend/tests/test_task_persistence.py +++ b/backend/tests/test_task_persistence.py @@ -6,16 +6,17 @@ # @TEST_DATA: valid_task -> {"id": "test-uuid-1", "plugin_id": "backup", "status": "PENDING"} # [SECTION: IMPORTS] -from datetime import datetime, timedelta +from datetime import datetime from unittest.mock import patch from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from src.core.task_manager.models import LogEntry, Task, TaskStatus +from src.core.task_manager.persistence import TaskPersistenceService from src.models.mapping import Base, Environment from src.models.task import TaskRecord -from src.core.task_manager.persistence import TaskPersistenceService -from src.core.task_manager.models import Task, TaskStatus, LogEntry + # [/SECTION] diff --git a/backend/tests/test_translate_corrections.py b/backend/tests/test_translate_corrections.py index 92c69f26..e447d41b 100644 --- a/backend/tests/test_translate_corrections.py +++ b/backend/tests/test_translate_corrections.py @@ -27,19 +27,17 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest from unittest.mock import MagicMock + +import pytest from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from datetime import datetime, timezone -import uuid -from src.core.database import Base from src.app import app -from src.dependencies import get_current_user, get_config_manager, has_permission, get_db +from src.core.database import Base +from src.dependencies import get_config_manager, get_current_user, get_db from src.plugins.translate.dictionary import DictionaryManager -from src.models.translate import TerminologyDictionary, DictionaryEntry # In-memory SQLite SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" diff --git a/backend/tests/test_translate_executor_filter.py b/backend/tests/test_translate_executor_filter.py index 11d5d1d8..ba929c91 100644 --- a/backend/tests/test_translate_executor_filter.py +++ b/backend/tests/test_translate_executor_filter.py @@ -14,12 +14,10 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest from unittest.mock import MagicMock from src.plugins.translate.executor import TranslationExecutor - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- diff --git a/backend/tests/test_translate_history.py b/backend/tests/test_translate_history.py index 697430c2..e55d3cef 100644 --- a/backend/tests/test_translate_history.py +++ b/backend/tests/test_translate_history.py @@ -26,20 +26,19 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) +import uuid +from datetime import UTC, datetime +from unittest.mock import MagicMock + import pytest -from unittest.mock import MagicMock, patch from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from datetime import datetime, timezone -import uuid -import json -from src.core.database import Base from src.app import app -from src.dependencies import get_current_user, get_config_manager, has_permission, get_db -from src.plugins.translate.metrics import TranslationMetrics -from src.models.translate import TranslationJob, TranslationRun, TranslationEvent, MetricSnapshot +from src.core.database import Base +from src.dependencies import get_config_manager, get_current_user, get_db +from src.models.translate import TranslationEvent, TranslationJob, TranslationRun # In-memory SQLite SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" @@ -130,7 +129,7 @@ def _create_test_run(db_session, job_id: str, status: str = "COMPLETED", trigger dict_snapshot_hash="def456", key_hash="ghi789", created_by="testuser", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) db_session.add(run) db_session.flush() @@ -197,7 +196,7 @@ def test_list_runs_filter_status(client, mock_api_deps): _create_test_run(session, job.id, status="FAILED") session.commit() - response = client.get(f"/api/translate/runs?run_status=FAILED") + response = client.get("/api/translate/runs?run_status=FAILED") assert response.status_code == 200 data = response.json() for item in data["items"]: @@ -221,7 +220,7 @@ def test_get_run_detail(client, mock_api_deps): event_type="RUN_STARTED", event_data={}, created_by="testuser", - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), ) session.add(event) session.commit() diff --git a/backend/tests/test_translate_jobs.py b/backend/tests/test_translate_jobs.py index ab296ad6..d8eb9e1b 100644 --- a/backend/tests/test_translate_jobs.py +++ b/backend/tests/test_translate_jobs.py @@ -29,26 +29,20 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) +from unittest.mock import MagicMock + import pytest -from unittest.mock import MagicMock, patch -from datetime import datetime, timezone -from fastapi import status from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from typing import Any, Dict, List, Optional, Tuple -import json -import uuid -from src.core.database import Base from src.app import app -from src.dependencies import ( - get_current_user, - get_config_manager, - has_permission, -) from src.core.config_models import Environment - +from src.core.database import Base +from src.dependencies import ( + get_config_manager, + get_current_user, +) # [DEF:valid_job_payload:Variable] # @PURPOSE: Standard valid payload for creating a translation job. diff --git a/backend/tests/test_translate_scheduler.py b/backend/tests/test_translate_scheduler.py index b1a77c6e..319842cd 100644 --- a/backend/tests/test_translate_scheduler.py +++ b/backend/tests/test_translate_scheduler.py @@ -26,20 +26,16 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest from unittest.mock import MagicMock + +import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from datetime import datetime, timezone -import uuid from src.core.database import Base -from src.app import app -from src.dependencies import get_current_user, get_config_manager, get_db -from src.plugins.translate.scheduler import TranslationScheduler, execute_scheduled_translation +from src.plugins.translate.scheduler import TranslationScheduler from src.plugins.translate.service import TranslateJobService from src.schemas.translate import TranslateJobCreate -from src.models.translate import TranslationJob, TranslationSchedule, TranslationRun # In-memory SQLite SQLALCHEMY_DATABASE_URL = "sqlite:///:memory:" @@ -63,7 +59,6 @@ def db_session(): # @PURPOSE: Verify schedule creation with valid params. def test_create_schedule(db_session): """Test creating a schedule for a job.""" - from src.core.config_models import Environment config_mgr = MagicMock() config_mgr.get_environments.return_value = [] @@ -90,7 +85,6 @@ def test_create_schedule(db_session): # @PURPOSE: Verify schedule update. def test_update_schedule(db_session): """Test updating a schedule.""" - from src.core.config_models import Environment config_mgr = MagicMock() config_mgr.get_environments.return_value = [] @@ -111,7 +105,6 @@ def test_update_schedule(db_session): # @PURPOSE: Verify schedule deletion. def test_delete_schedule(db_session): """Test deleting a schedule.""" - from src.core.config_models import Environment config_mgr = MagicMock() config_mgr.get_environments.return_value = [] @@ -131,7 +124,6 @@ def test_delete_schedule(db_session): # @PURPOSE: Verify enable/disable toggle. def test_enable_disable_schedule(db_session): """Test enabling and disabling a schedule.""" - from src.core.config_models import Environment config_mgr = MagicMock() config_mgr.get_environments.return_value = [] @@ -153,7 +145,6 @@ def test_enable_disable_schedule(db_session): # @PURPOSE: Verify ValueError on getting non-existent schedule. def test_get_schedule_not_found(db_session): """Test that getting a non-existent schedule raises ValueError.""" - from src.core.config_models import Environment config_mgr = MagicMock() config_mgr.get_environments.return_value = [] @@ -167,7 +158,6 @@ def test_get_schedule_not_found(db_session): # @PURPOSE: Verify listing only active schedules. def test_list_active_schedules(db_session): """Test listing active schedules.""" - from src.core.config_models import Environment config_mgr = MagicMock() config_mgr.get_environments.return_value = [] diff --git a/backend/tests/test_translate_scheduler_execution.py b/backend/tests/test_translate_scheduler_execution.py index 5f64cc70..5fe637a6 100644 --- a/backend/tests/test_translate_scheduler_execution.py +++ b/backend/tests/test_translate_scheduler_execution.py @@ -9,10 +9,10 @@ # @TEST_INVARIANT: trigger_type_dispatch -> VERIFIED_BY: [test_new_key_only_mode, test_baseline_expired_fallback, test_full_mode_default] import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -from unittest.mock import MagicMock, patch -from datetime import datetime, timezone, timedelta +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) +from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock, patch # -- Helpers ----------------------------------------------------------------- @@ -23,7 +23,7 @@ def _make_most_recent_run(created_at_delta_days=60): run.job_id = "job-1" run.status = "COMPLETED" run.insert_status = "succeeded" - run.created_at = datetime.now(timezone.utc) - timedelta(days=created_at_delta_days) + run.created_at = datetime.now(UTC) - timedelta(days=created_at_delta_days) return run diff --git a/backend/tests/test_translate_scheduler_guard.py b/backend/tests/test_translate_scheduler_guard.py index ee0b8ed2..96e95bd7 100644 --- a/backend/tests/test_translate_scheduler_guard.py +++ b/backend/tests/test_translate_scheduler_guard.py @@ -13,10 +13,8 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -import pytest +from datetime import UTC, datetime, timedelta from unittest.mock import MagicMock, patch -from datetime import datetime, timezone, timedelta - # --------------------------------------------------------------------------- # Helpers @@ -29,7 +27,7 @@ def _make_most_recent_run(created_at_delta_days=60): run.job_id = "job-1" run.status = "COMPLETED" run.insert_status = "succeeded" - run.created_at = datetime.now(timezone.utc) - timedelta(days=created_at_delta_days) + run.created_at = datetime.now(UTC) - timedelta(days=created_at_delta_days) return run @@ -85,20 +83,19 @@ def test_concurrent_run_skips(): with patch( "src.plugins.translate.orchestrator.TranslationOrchestrator" - ) as mock_orch_cls: - with patch( - "src.plugins.translate.scheduler.TranslationEventLog" - ) as mock_event_log_cls: - mock_event_log = MagicMock() - mock_event_log_cls.return_value = mock_event_log + ) as mock_orch_cls, patch( + "src.plugins.translate.scheduler.TranslationEventLog" + ) as mock_event_log_cls: + mock_event_log = MagicMock() + mock_event_log_cls.return_value = mock_event_log - execute_scheduled_translation( - schedule_id="sched-1", - job_id="job-1", - db_session_maker=mock_session_maker, - config_manager=mock_config, - execution_mode="full", - ) + execute_scheduled_translation( + schedule_id="sched-1", + job_id="job-1", + db_session_maker=mock_session_maker, + config_manager=mock_config, + execution_mode="full", + ) mock_orch_cls.assert_not_called() mock_event_log.log_event.assert_called_once() @@ -128,7 +125,7 @@ def test_recent_pending_run_not_stale(): pending_run.id = "pending-run-1" pending_run.job_id = "job-1" pending_run.status = "PENDING" - pending_run.created_at = datetime.now(timezone.utc) - timedelta(minutes=30) + pending_run.created_at = datetime.now(UTC) - timedelta(minutes=30) q1 = MagicMock() q1.filter.return_value.first.return_value = schedule @@ -140,20 +137,19 @@ def test_recent_pending_run_not_stale(): with patch( "src.plugins.translate.orchestrator.TranslationOrchestrator" - ) as mock_orch_cls: - with patch( - "src.plugins.translate.scheduler.TranslationEventLog" - ) as mock_event_log_cls: - mock_event_log = MagicMock() - mock_event_log_cls.return_value = mock_event_log + ) as mock_orch_cls, patch( + "src.plugins.translate.scheduler.TranslationEventLog" + ) as mock_event_log_cls: + mock_event_log = MagicMock() + mock_event_log_cls.return_value = mock_event_log - execute_scheduled_translation( - schedule_id="sched-1", - job_id="job-1", - db_session_maker=mock_session_maker, - config_manager=mock_config, - execution_mode="full", - ) + execute_scheduled_translation( + schedule_id="sched-1", + job_id="job-1", + db_session_maker=mock_session_maker, + config_manager=mock_config, + execution_mode="full", + ) mock_orch_cls.assert_not_called() mock_event_log.log_event.assert_called_once() @@ -183,7 +179,7 @@ def test_stale_pending_cleared_and_proceeds(): stale_run.id = "stale-run-1" stale_run.job_id = "job-1" stale_run.status = "PENDING" - stale_run.created_at = datetime.now(timezone.utc) - timedelta(hours=2) + stale_run.created_at = datetime.now(UTC) - timedelta(hours=2) most_recent = _make_most_recent_run(created_at_delta_days=30) @@ -248,13 +244,13 @@ def test_multiple_stale_pending_all_cleared(): stale_run_a.id = "stale-run-a" stale_run_a.job_id = "job-1" stale_run_a.status = "PENDING" - stale_run_a.created_at = datetime.now(timezone.utc) - timedelta(hours=5) + stale_run_a.created_at = datetime.now(UTC) - timedelta(hours=5) stale_run_b = MagicMock() stale_run_b.id = "stale-run-b" stale_run_b.job_id = "job-1" stale_run_b.status = "PENDING" - stale_run_b.created_at = datetime.now(timezone.utc) - timedelta(hours=3) + stale_run_b.created_at = datetime.now(UTC) - timedelta(hours=3) most_recent = _make_most_recent_run(created_at_delta_days=30)